This is an automated email from the ASF dual-hosted git repository. rakeshr pushed a commit to branch HDDS-2939 in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 0edc68c8185516e5d2f188e439d4d57042f9921f Author: Rakesh Radhakrishnan <[email protected]> AuthorDate: Mon Jan 11 20:03:55 2021 +0530 HDDS-2942. Putkey : create key table entries for intermediate directories in the key path (#1764) --- .../apache/hadoop/ozone/om/TestObjectStoreV1.java | 215 +++++++++++++++++++++ .../om/ratis/utils/OzoneManagerRatisUtils.java | 4 + .../om/request/file/OMFileCreateRequestV1.java | 3 +- .../ozone/om/request/key/OMKeyCommitRequestV1.java | 5 +- .../ozone/om/request/key/OMKeyCreateRequest.java | 22 ++- .../OMKeyCreateRequestV1.java} | 148 ++++++-------- .../om/response/file/OMFileCreateResponseV1.java | 3 +- .../om/response/key/OMAllocateBlockResponseV1.java | 1 - .../om/response/key/OMKeyCommitResponseV1.java | 2 - ...kResponseV1.java => OMKeyCreateResponseV1.java} | 42 ++-- .../om/request/key/TestOMKeyCreateRequest.java | 53 ++--- .../om/request/key/TestOMKeyCreateRequestV1.java | 129 +++++++++++++ .../key/TestOMAllocateBlockResponseV1.java | 5 +- .../om/response/key/TestOMKeyCommitResponseV1.java | 3 +- ...ponseV1.java => TestOMKeyCreateResponseV1.java} | 77 +++----- 15 files changed, 501 insertions(+), 211 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreV1.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreV1.java new file mode 100644 index 0000000..c6ae4ca --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreV1.java @@ -0,0 +1,215 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.om; + +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.hadoop.hdds.client.ReplicationFactor; +import org.apache.hadoop.hdds.client.ReplicationType; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.TableIterator; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.client.ObjectStore; +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.ozone.client.io.OzoneOutputStream; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; +import org.apache.hadoop.util.StringUtils; +import org.junit.Assert; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; + +import java.io.IOException; +import java.util.HashMap; +import java.util.UUID; + +public class TestObjectStoreV1 { + + private static MiniOzoneCluster cluster = null; + private static OzoneConfiguration conf; + private static String clusterId; + private static String scmId; + private static String omId; + + @Rule + public Timeout timeout = new Timeout(240000); + + /** + * Create a MiniDFSCluster for testing. + * <p> + * + * @throws IOException + */ + @BeforeClass + public static void init() throws Exception { + conf = new OzoneConfiguration(); + clusterId = UUID.randomUUID().toString(); + scmId = UUID.randomUUID().toString(); + omId = UUID.randomUUID().toString(); + conf.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); + cluster = MiniOzoneCluster.newBuilder(conf) + .setClusterId(clusterId) + .setScmId(scmId) + .setOmId(omId) + .build(); + cluster.waitForClusterToBeReady(); + } + + @Test + public void testCreateKey() throws Exception { + String volumeName = "volume" + RandomStringUtils.randomNumeric(5); + String bucketName = "bucket" + RandomStringUtils.randomNumeric(5); + String parent = "a/b/c/"; + String file = "key" + RandomStringUtils.randomNumeric(5); + String key = parent + file; + + OzoneClient client = cluster.getClient(); + + ObjectStore objectStore = client.getObjectStore(); + objectStore.createVolume(volumeName); + + OzoneVolume ozoneVolume = objectStore.getVolume(volumeName); + Assert.assertTrue(ozoneVolume.getName().equals(volumeName)); + ozoneVolume.createBucket(bucketName); + + OzoneBucket ozoneBucket = ozoneVolume.getBucket(bucketName); + Assert.assertTrue(ozoneBucket.getName().equals(bucketName)); + + Table<String, OmKeyInfo> openKeyTable = + cluster.getOzoneManager().getMetadataManager().getOpenKeyTable(); + + // before file creation + verifyKeyInFileTable(openKeyTable, file, 0, true); + + String data = "random data"; + OzoneOutputStream ozoneOutputStream = ozoneBucket.createKey(key, + data.length(), ReplicationType.RATIS, ReplicationFactor.ONE, + new HashMap<>()); + + OmDirectoryInfo dirPathC = getDirInfo(volumeName, bucketName, parent); + Assert.assertNotNull("Failed to find dir path: a/b/c", dirPathC); + + // after file creation + verifyKeyInOpenFileTable(openKeyTable, file, dirPathC.getObjectID(), + false); + + ozoneOutputStream.write(data.getBytes(), 0, data.length()); + ozoneOutputStream.close(); + + Table<String, OmKeyInfo> keyTable = + cluster.getOzoneManager().getMetadataManager().getKeyTable(); + + // After closing the file. File entry should be removed from openFileTable + // and it should be added to fileTable. + verifyKeyInFileTable(keyTable, file, dirPathC.getObjectID(), false); + verifyKeyInOpenFileTable(openKeyTable, file, dirPathC.getObjectID(), + true); + + ozoneBucket.deleteKey(key); + + // after key delete + verifyKeyInFileTable(keyTable, file, dirPathC.getObjectID(), true); + verifyKeyInOpenFileTable(openKeyTable, file, dirPathC.getObjectID(), + true); + } + + private OmDirectoryInfo getDirInfo(String volumeName, String bucketName, + String parentKey) throws Exception { + OMMetadataManager omMetadataManager = + cluster.getOzoneManager().getMetadataManager(); + long bucketId = TestOMRequestUtils.getBucketId(volumeName, bucketName, + omMetadataManager); + String[] pathComponents = StringUtils.split(parentKey, '/'); + long parentId = bucketId; + OmDirectoryInfo dirInfo = null; + for (int indx = 0; indx < pathComponents.length; indx++) { + String pathElement = pathComponents[indx]; + String dbKey = omMetadataManager.getOzonePathKey(parentId, + pathElement); + dirInfo = + omMetadataManager.getDirectoryTable().get(dbKey); + parentId = dirInfo.getObjectID(); + } + return dirInfo; + } + + private void verifyKeyInFileTable(Table<String, OmKeyInfo> fileTable, + String fileName, long parentID, boolean isEmpty) throws IOException { + TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>> iterator + = fileTable.iterator(); + + if (isEmpty) { + Assert.assertTrue("Table is not empty!", fileTable.isEmpty()); + } else { + Assert.assertFalse("Table is empty!", fileTable.isEmpty()); + while (iterator.hasNext()) { + Table.KeyValue<String, OmKeyInfo> next = iterator.next(); + Assert.assertEquals("Invalid Key: " + next.getKey(), + parentID + "/" + fileName, next.getKey()); + OmKeyInfo omKeyInfo = next.getValue(); + Assert.assertEquals("Invalid Key", fileName, + omKeyInfo.getFileName()); + Assert.assertEquals("Invalid Key", fileName, + omKeyInfo.getKeyName()); + Assert.assertEquals("Invalid Key", parentID, + omKeyInfo.getParentObjectID()); + } + } + } + + private void verifyKeyInOpenFileTable(Table<String, OmKeyInfo> openFileTable, + String fileName, long parentID, boolean isEmpty) throws IOException { + TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>> iterator + = openFileTable.iterator(); + + if (isEmpty) { + Assert.assertTrue("Table is not empty!", openFileTable.isEmpty()); + } else { + Assert.assertFalse("Table is empty!", openFileTable.isEmpty()); + while (iterator.hasNext()) { + Table.KeyValue<String, OmKeyInfo> next = iterator.next(); + // used startsWith because the key format is, + // <parentID>/fileName/<clientID> and clientID is not visible. + Assert.assertTrue("Invalid Key: " + next.getKey(), + next.getKey().startsWith(parentID + "/" + fileName)); + OmKeyInfo omKeyInfo = next.getValue(); + Assert.assertEquals("Invalid Key", fileName, + omKeyInfo.getFileName()); + Assert.assertEquals("Invalid Key", fileName, + omKeyInfo.getKeyName()); + Assert.assertEquals("Invalid Key", parentID, + omKeyInfo.getParentObjectID()); + } + } + } + + /** + * Shutdown MiniDFSCluster. + */ + @AfterClass + public static void shutdown() { + if (cluster != null) { + cluster.shutdown(); + } + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java index 31120f1..c469c2c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java @@ -42,6 +42,7 @@ import org.apache.hadoop.ozone.om.request.key.OMAllocateBlockRequestV1; import org.apache.hadoop.ozone.om.request.key.OMKeyCommitRequest; import org.apache.hadoop.ozone.om.request.key.OMKeyCommitRequestV1; import org.apache.hadoop.ozone.om.request.key.OMKeyCreateRequest; +import org.apache.hadoop.ozone.om.request.key.OMKeyCreateRequestV1; import org.apache.hadoop.ozone.om.request.key.OMKeyDeleteRequest; import org.apache.hadoop.ozone.om.request.key.OMKeyDeleteRequestV1; import org.apache.hadoop.ozone.om.request.key.OMKeyPurgeRequest; @@ -144,6 +145,9 @@ public final class OzoneManagerRatisUtils { } return new OMAllocateBlockRequest(omRequest); case CreateKey: + if (omLayoutVersionV1) { + return new OMKeyCreateRequestV1(omRequest); + } return new OMKeyCreateRequest(omRequest); case CommitKey: if (omLayoutVersionV1) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequestV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequestV1.java index e38908a..f35c9a5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequestV1.java @@ -217,7 +217,8 @@ public class OMFileCreateRequestV1 extends OMFileCreateRequest { .setOpenVersion(openVersion).build()) .setCmdType(Type.CreateFile); omClientResponse = new OMFileCreateResponseV1(omResponse.build(), - omFileInfo, missingParentInfos, clientID, omBucketInfo.copyObject()); + omFileInfo, missingParentInfos, clientID, + omBucketInfo.copyObject()); result = Result.SUCCESS; } catch (IOException ex) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestV1.java index 7d99119..64991de 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestV1.java @@ -25,7 +25,6 @@ import org.apache.hadoop.ozone.om.OMMetrics; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; -import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; @@ -91,7 +90,6 @@ public class OMKeyCommitRequestV1 extends OMKeyCommitRequest { IOException exception = null; OmKeyInfo omKeyInfo = null; - OmVolumeArgs omVolumeArgs = null; OmBucketInfo omBucketInfo = null; OMClientResponse omClientResponse = null; boolean bucketLockAcquired = false; @@ -168,8 +166,7 @@ public class OMKeyCommitRequestV1 extends OMKeyCommitRequest { omBucketInfo.incrUsedBytes(correctedSpace); omClientResponse = new OMKeyCommitResponseV1(omResponse.build(), - omKeyInfo, dbFileKey, dbOpenFileKey, omVolumeArgs, - omBucketInfo.copyObject()); + omKeyInfo, dbFileKey, dbOpenFileKey, omBucketInfo.copyObject()); result = Result.SUCCESS; } catch (IOException ex) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java index 55f4990..68c0c36 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java @@ -349,24 +349,34 @@ public class OMKeyCreateRequest extends OMKeyRequest { OMAction.ALLOCATE_KEY, auditMap, exception, getOmRequest().getUserInfo())); + logResult(createKeyRequest, omMetrics, exception, result, + numMissingParents); + + return omClientResponse; + } + + protected void logResult(CreateKeyRequest createKeyRequest, + OMMetrics omMetrics, IOException exception, Result result, + int numMissingParents) { switch (result) { case SUCCESS: // Missing directories are created immediately, counting that here. // The metric for the key is incremented as part of the key commit. omMetrics.incNumKeys(numMissingParents); - LOG.debug("Key created. Volume:{}, Bucket:{}, Key:{}", volumeName, - bucketName, keyName); + LOG.debug("Key created. Volume:{}, Bucket:{}, Key:{}", + createKeyRequest.getKeyArgs().getVolumeName(), + createKeyRequest.getKeyArgs().getBucketName(), + createKeyRequest.getKeyArgs().getKeyName()); break; case FAILURE: LOG.error("Key creation failed. Volume:{}, Bucket:{}, Key{}. " + - "Exception:{}", volumeName, bucketName, keyName, exception); + "Exception:{}", createKeyRequest.getKeyArgs().getVolumeName(), + createKeyRequest.getKeyArgs().getBucketName(), + createKeyRequest.getKeyArgs().getKeyName(), exception); break; default: LOG.error("Unrecognized Result for OMKeyCreateRequest: {}", createKeyRequest); } - - return omClientResponse; } - } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequestV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequestV1.java similarity index 66% copy from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequestV1.java copy to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequestV1.java index e38908a..416e462 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequestV1.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.ozone.om.request.file; +package org.apache.hadoop.ozone.om.request.key; import com.google.common.base.Optional; import org.apache.hadoop.ozone.audit.OMAction; @@ -29,15 +29,15 @@ import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; +import org.apache.hadoop.ozone.om.request.file.OMDirectoryCreateRequestV1; +import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.response.OMClientResponse; -import org.apache.hadoop.ozone.om.response.file.OMFileCreateResponse; -import org.apache.hadoop.ozone.om.response.file.OMFileCreateResponseV1; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateFileRequest; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateFileResponse; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.apache.hadoop.ozone.om.response.key.OMKeyCreateResponseV1; +import org.apache.hadoop.ozone.om.response.key.OMKeyCreateResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateKeyResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; import org.apache.hadoop.ozone.security.acl.OzoneObj; @@ -51,16 +51,19 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_A_FILE; import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK; +import static org.apache.hadoop.ozone.om.request.file.OMFileRequest.OMDirectoryResult.DIRECTORY_EXISTS; +import static org.apache.hadoop.ozone.om.request.file.OMFileRequest.OMDirectoryResult.FILE_EXISTS_IN_GIVENPATH; /** - * Handles create file request layout version1. + * Handles CreateKey request layout version1. */ -public class OMFileCreateRequestV1 extends OMFileCreateRequest { - +public class OMKeyCreateRequestV1 extends OMKeyCreateRequest { private static final Logger LOG = - LoggerFactory.getLogger(OMFileCreateRequestV1.class); - public OMFileCreateRequestV1(OMRequest omRequest) { + LoggerFactory.getLogger(OMKeyCreateRequestV1.class); + + public OMKeyCreateRequestV1(OMRequest omRequest) { super(omRequest); } @@ -69,61 +72,42 @@ public class OMFileCreateRequestV1 extends OMFileCreateRequest { public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, long trxnLogIndex, OzoneManagerDoubleBufferHelper omDoubleBufferHelper) { - CreateFileRequest createFileRequest = getOmRequest().getCreateFileRequest(); - KeyArgs keyArgs = createFileRequest.getKeyArgs(); + OzoneManagerProtocolProtos.CreateKeyRequest createKeyRequest = + getOmRequest().getCreateKeyRequest(); + + OzoneManagerProtocolProtos.KeyArgs keyArgs = createKeyRequest.getKeyArgs(); Map<String, String> auditMap = buildKeyArgsAuditMap(keyArgs); String volumeName = keyArgs.getVolumeName(); String bucketName = keyArgs.getBucketName(); String keyName = keyArgs.getKeyName(); - // if isRecursive is true, file would be created even if parent - // directories does not exist. - boolean isRecursive = createFileRequest.getIsRecursive(); - if (LOG.isDebugEnabled()) { - LOG.debug("File create for : " + volumeName + "/" + bucketName + "/" - + keyName + ":" + isRecursive); - } - - // if isOverWrite is true, file would be over written. - boolean isOverWrite = createFileRequest.getIsOverwrite(); - OMMetrics omMetrics = ozoneManager.getMetrics(); - omMetrics.incNumCreateFile(); + omMetrics.incNumKeyAllocates(); OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager(); - - boolean acquiredLock = false; - OmBucketInfo omBucketInfo = null; final List<OmKeyLocationInfo> locations = new ArrayList<>(); - List<OmDirectoryInfo> missingParentInfos; - int numKeysCreated = 0; + boolean acquireLock = false; OMClientResponse omClientResponse = null; - OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder( - getOmRequest()); + OzoneManagerProtocolProtos.OMResponse.Builder omResponse = + OmResponseUtil.getOMResponseBuilder(getOmRequest()); IOException exception = null; - Result result = null; + Result result; + List<OmDirectoryInfo> missingParentInfos; + int numKeysCreated = 0; try { keyArgs = resolveBucketLink(ozoneManager, keyArgs, auditMap); volumeName = keyArgs.getVolumeName(); bucketName = keyArgs.getBucketName(); - if (keyName.length() == 0) { - // Check if this is the root of the filesystem. - throw new OMException("Can not write to directory: " + keyName, - OMException.ResultCodes.NOT_A_FILE); - } - // check Acl checkKeyAcls(ozoneManager, volumeName, bucketName, keyName, - IAccessAuthorizer.ACLType.CREATE, OzoneObj.ResourceType.KEY); - - // acquire lock - acquiredLock = omMetadataManager.getLock().acquireWriteLock(BUCKET_LOCK, - volumeName, bucketName); + IAccessAuthorizer.ACLType.CREATE, OzoneObj.ResourceType.KEY); + acquireLock = omMetadataManager.getLock().acquireWriteLock(BUCKET_LOCK, + volumeName, bucketName); validateBucketAndVolume(omMetadataManager, volumeName, bucketName); OmKeyInfo dbFileInfo = null; @@ -144,12 +128,14 @@ public class OMFileCreateRequestV1 extends OMFileCreateRequest { } } - // check if the file or directory already existed in OM - checkDirectoryResult(keyName, isOverWrite, - pathInfoV1.getDirectoryResult()); - - if (!isRecursive) { - checkAllParentsExist(keyArgs, pathInfoV1); + // Check if a file or directory exists with same key name. + if (pathInfoV1.getDirectoryResult() == DIRECTORY_EXISTS) { + throw new OMException("Cannot write to " + + "directory. createIntermediateDirs behavior is enabled and " + + "hence / has special interpretation: " + keyName, NOT_A_FILE); + } else if (pathInfoV1.getDirectoryResult() == FILE_EXISTS_IN_GIVENPATH) { + throw new OMException("Can not create file: " + keyName + + " as there is already file in the given path", NOT_A_FILE); } // add all missing parents to dir table @@ -162,7 +148,7 @@ public class OMFileCreateRequestV1 extends OMFileCreateRequest { // do open key OmBucketInfo bucketInfo = omMetadataManager.getBucketTable().get( - omMetadataManager.getBucketKey(volumeName, bucketName)); + omMetadataManager.getBucketKey(volumeName, bucketName)); OmKeyInfo omFileInfo = prepareFileInfo(omMetadataManager, keyArgs, dbFileInfo, keyArgs.getDataSize(), locations, @@ -172,15 +158,15 @@ public class OMFileCreateRequestV1 extends OMFileCreateRequest { ozoneManager.isRatisEnabled()); long openVersion = omFileInfo.getLatestVersionLocations().getVersion(); - long clientID = createFileRequest.getClientID(); + long clientID = createKeyRequest.getClientID(); String dbOpenFileName = omMetadataManager.getOpenFileName( pathInfoV1.getLastKnownParentId(), pathInfoV1.getLeafNodeName(), clientID); // Append new blocks List<OmKeyLocationInfo> newLocationList = keyArgs.getKeyLocationsList() - .stream().map(OmKeyLocationInfo::getFromProtobuf) - .collect(Collectors.toList()); + .stream().map(OmKeyLocationInfo::getFromProtobuf) + .collect(Collectors.toList()); omFileInfo.appendNewBlocks(newLocationList, false); omBucketInfo = getBucketInfo(omMetadataManager, volumeName, bucketName); @@ -210,51 +196,39 @@ public class OMFileCreateRequestV1 extends OMFileCreateRequest { // Prepare response. Sets user given full key name in the 'keyName' // attribute in response object. - int clientVersion = getOmRequest().getVersion(); - omResponse.setCreateFileResponse(CreateFileResponse.newBuilder() - .setKeyInfo(omFileInfo.getProtobuf(keyName, clientVersion)) - .setID(clientID) - .setOpenVersion(openVersion).build()) - .setCmdType(Type.CreateFile); - omClientResponse = new OMFileCreateResponseV1(omResponse.build(), - omFileInfo, missingParentInfos, clientID, omBucketInfo.copyObject()); + omResponse.setCreateKeyResponse(CreateKeyResponse.newBuilder() + .setKeyInfo(omFileInfo.getProtobuf(keyName)) + .setID(clientID) + .setOpenVersion(openVersion).build()) + .setCmdType(Type.CreateKey); + omClientResponse = new OMKeyCreateResponseV1(omResponse.build(), + omFileInfo, missingParentInfos, clientID, + omBucketInfo.copyObject()); result = Result.SUCCESS; } catch (IOException ex) { result = Result.FAILURE; exception = ex; - omMetrics.incNumCreateFileFails(); - omResponse.setCmdType(Type.CreateFile); - omClientResponse = new OMFileCreateResponse(createErrorOMResponse( - omResponse, exception)); + omMetrics.incNumKeyAllocateFails(); + omResponse.setCmdType(Type.CreateKey); + omClientResponse = new OMKeyCreateResponse( + createErrorOMResponse(omResponse, exception)); } finally { addResponseToDoubleBuffer(trxnLogIndex, omClientResponse, - omDoubleBufferHelper); - if (acquiredLock) { + omDoubleBufferHelper); + if (acquireLock) { omMetadataManager.getLock().releaseWriteLock(BUCKET_LOCK, volumeName, - bucketName); + bucketName); } } // Audit Log outside the lock auditLog(ozoneManager.getAuditLogger(), buildAuditMessage( - OMAction.CREATE_FILE, auditMap, exception, - getOmRequest().getUserInfo())); - - switch (result) { - case SUCCESS: - omMetrics.incNumKeys(numKeysCreated); - LOG.debug("File created. Volume:{}, Bucket:{}, Key:{}", volumeName, - bucketName, keyName); - break; - case FAILURE: - LOG.error("File create failed. Volume:{}, Bucket:{}, Key{}.", - volumeName, bucketName, keyName, exception); - break; - default: - LOG.error("Unrecognized Result for OMFileCreateRequest: {}", - createFileRequest); - } + OMAction.ALLOCATE_KEY, auditMap, exception, + getOmRequest().getUserInfo())); + + logResult(createKeyRequest, omMetrics, exception, result, + numKeysCreated); return omClientResponse; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMFileCreateResponseV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMFileCreateResponseV1.java index ccaaa6b..7325def 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMFileCreateResponseV1.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMFileCreateResponseV1.java @@ -32,12 +32,13 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DIRECTORY_TABLE; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.OPEN_FILE_TABLE; /** * Response for create file request layout version V1. */ -@CleanupTableInfo(cleanupTables = OPEN_FILE_TABLE) +@CleanupTableInfo(cleanupTables = {DIRECTORY_TABLE, OPEN_FILE_TABLE}) public class OMFileCreateResponseV1 extends OMFileCreateResponse { private List<OmDirectoryInfo> parentDirInfos; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponseV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponseV1.java index ef8b639..138cca1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponseV1.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponseV1.java @@ -22,7 +22,6 @@ import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCommitResponseV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCommitResponseV1.java index c0840e3..5f0a337 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCommitResponseV1.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCommitResponseV1.java @@ -22,7 +22,6 @@ import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; @@ -43,7 +42,6 @@ public class OMKeyCommitResponseV1 extends OMKeyCommitResponse { public OMKeyCommitResponseV1(@Nonnull OMResponse omResponse, @Nonnull OmKeyInfo omKeyInfo, String ozoneKeyName, String openKeyName, - @Nonnull OmVolumeArgs omVolumeArgs, @Nonnull OmBucketInfo omBucketInfo) { super(omResponse, omKeyInfo, ozoneKeyName, openKeyName, omBucketInfo); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponseV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCreateResponseV1.java similarity index 51% copy from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponseV1.java copy to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCreateResponseV1.java index ef8b639..59c7edf 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponseV1.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCreateResponseV1.java @@ -18,43 +18,31 @@ package org.apache.hadoop.ozone.om.response.key; -import org.apache.hadoop.hdds.utils.db.BatchOperation; -import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; -import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; +import org.apache.hadoop.ozone.om.response.file.OMFileCreateResponseV1; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; import javax.annotation.Nonnull; -import java.io.IOException; +import java.util.List; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DIRECTORY_TABLE; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.OPEN_FILE_TABLE; /** - * Response for AllocateBlock request layout version V1. + * Response for CreateKey request layout version V1. */ -@CleanupTableInfo(cleanupTables = {OPEN_FILE_TABLE}) -public class OMAllocateBlockResponseV1 extends OMAllocateBlockResponse { - - public OMAllocateBlockResponseV1(@Nonnull OMResponse omResponse, - @Nonnull OmKeyInfo omKeyInfo, long clientID, - @Nonnull OmBucketInfo omBucketInfo) { - super(omResponse, omKeyInfo, clientID, omBucketInfo); - } - - @Override - public void addToDBBatch(OMMetadataManager omMetadataManager, - BatchOperation batchOperation) throws IOException { - - OMFileRequest.addToOpenFileTable(omMetadataManager, batchOperation, - getOmKeyInfo(), getClientID()); - - // update bucket usedBytes. - omMetadataManager.getBucketTable().putWithBatch(batchOperation, - omMetadataManager.getBucketKey(getOmKeyInfo().getVolumeName(), - getOmKeyInfo().getBucketName()), getOmBucketInfo()); +@CleanupTableInfo(cleanupTables = {DIRECTORY_TABLE, OPEN_FILE_TABLE}) +public class OMKeyCreateResponseV1 extends OMFileCreateResponseV1 { + + public OMKeyCreateResponseV1(@Nonnull OMResponse omResponse, + @Nonnull OmKeyInfo omKeyInfo, + @Nonnull List<OmDirectoryInfo> parentDirInfos, + long openKeySessionID, + @Nonnull OmBucketInfo omBucketInfo) { + super(omResponse, omKeyInfo, parentDirInfos, openKeySessionID, + omBucketInfo); } } - diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java index 7269957..3df6f38 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java @@ -74,7 +74,7 @@ public class TestOMKeyCreateRequest extends TestOMKeyRequest { doPreExecute(createKeyRequest(false, 0)); OMKeyCreateRequest omKeyCreateRequest = - new OMKeyCreateRequest(modifiedOmRequest); + getOMKeyCreateRequest(modifiedOmRequest); // Add volume and bucket entries to DB. addVolumeAndBucketToDB(volumeName, bucketName, @@ -82,8 +82,7 @@ public class TestOMKeyCreateRequest extends TestOMKeyRequest { long id = modifiedOmRequest.getCreateKeyRequest().getClientID(); - String openKey = omMetadataManager.getOpenKey(volumeName, bucketName, - keyName, id); + String openKey = getOpenKey(id); // Before calling OmKeyInfo omKeyInfo = omMetadataManager.getOpenKeyTable().get(openKey); @@ -138,7 +137,7 @@ public class TestOMKeyCreateRequest extends TestOMKeyRequest { doPreExecute(createKeyRequest(true, partNumber)); OMKeyCreateRequest omKeyCreateRequest = - new OMKeyCreateRequest(modifiedOmRequest); + getOMKeyCreateRequest(modifiedOmRequest); // Add volume and bucket entries to DB. addVolumeAndBucketToDB(volumeName, bucketName, @@ -178,7 +177,7 @@ public class TestOMKeyCreateRequest extends TestOMKeyRequest { doPreExecute(createKeyRequest(false, 0)); OMKeyCreateRequest omKeyCreateRequest = - new OMKeyCreateRequest(modifiedOmRequest); + getOMKeyCreateRequest(modifiedOmRequest); long id = modifiedOmRequest.getCreateKeyRequest().getClientID(); @@ -217,13 +216,12 @@ public class TestOMKeyCreateRequest extends TestOMKeyRequest { false, 0)); OMKeyCreateRequest omKeyCreateRequest = - new OMKeyCreateRequest(modifiedOmRequest); + getOMKeyCreateRequest(modifiedOmRequest); long id = modifiedOmRequest.getCreateKeyRequest().getClientID(); - String openKey = omMetadataManager.getOpenKey(volumeName, bucketName, - keyName, id); + String openKey = getOpenKey(id); TestOMRequestUtils.addVolumeToDB(volumeName, OzoneConsts.OZONE, omMetadataManager); @@ -248,8 +246,6 @@ public class TestOMKeyCreateRequest extends TestOMKeyRequest { } - - /** * This method calls preExecute and verify the modified request. * @param originalOMRequest @@ -259,7 +255,7 @@ public class TestOMKeyCreateRequest extends TestOMKeyRequest { private OMRequest doPreExecute(OMRequest originalOMRequest) throws Exception { OMKeyCreateRequest omKeyCreateRequest = - new OMKeyCreateRequest(originalOMRequest); + getOMKeyCreateRequest(originalOMRequest); OMRequest modifiedOmRequest = omKeyCreateRequest.preExecute(ozoneManager); @@ -349,7 +345,7 @@ public class TestOMKeyCreateRequest extends TestOMKeyRequest { @Test public void testKeyCreateWithFileSystemPathsEnabled() throws Exception { - OzoneConfiguration configuration = new OzoneConfiguration(); + OzoneConfiguration configuration = getOzoneConfiguration(); configuration.setBoolean(OZONE_OM_ENABLE_FILESYSTEM_PATHS, true); when(ozoneManager.getConfiguration()).thenReturn(configuration); when(ozoneManager.getEnableFileSystemPaths()).thenReturn(true); @@ -367,8 +363,7 @@ public class TestOMKeyCreateRequest extends TestOMKeyRequest { createAndCheck(keyName); // Commit openKey entry. - TestOMRequestUtils.addKeyToTable(false, volumeName, bucketName, - keyName.substring(1), 0L, RATIS, THREE, omMetadataManager); + addToKeyTable(keyName); // Now create another file in same dir path. keyName = "/a/b/c/file2"; @@ -430,10 +425,15 @@ public class TestOMKeyCreateRequest extends TestOMKeyRequest { } + protected void addToKeyTable(String keyName) throws Exception { + TestOMRequestUtils.addKeyToTable(false, volumeName, bucketName, + keyName.substring(1), 0L, RATIS, THREE, omMetadataManager); + } + private void checkNotAValidPath(String keyName) { OMRequest omRequest = createKeyRequest(false, 0, keyName); - OMKeyCreateRequest omKeyCreateRequest = new OMKeyCreateRequest(omRequest); + OMKeyCreateRequest omKeyCreateRequest = getOMKeyCreateRequest(omRequest); try { omKeyCreateRequest.preExecute(ozoneManager); @@ -450,11 +450,11 @@ public class TestOMKeyCreateRequest extends TestOMKeyRequest { private void checkNotAFile(String keyName) throws Exception { OMRequest omRequest = createKeyRequest(false, 0, keyName); - OMKeyCreateRequest omKeyCreateRequest = new OMKeyCreateRequest(omRequest); + OMKeyCreateRequest omKeyCreateRequest = getOMKeyCreateRequest(omRequest); omRequest = omKeyCreateRequest.preExecute(ozoneManager); - omKeyCreateRequest = new OMKeyCreateRequest(omRequest); + omKeyCreateRequest = getOMKeyCreateRequest(omRequest); OMClientResponse omClientResponse = omKeyCreateRequest.validateAndUpdateCache(ozoneManager, @@ -468,11 +468,11 @@ public class TestOMKeyCreateRequest extends TestOMKeyRequest { private void createAndCheck(String keyName) throws Exception { OMRequest omRequest = createKeyRequest(false, 0, keyName); - OMKeyCreateRequest omKeyCreateRequest = new OMKeyCreateRequest(omRequest); + OMKeyCreateRequest omKeyCreateRequest = getOMKeyCreateRequest(omRequest); omRequest = omKeyCreateRequest.preExecute(ozoneManager); - omKeyCreateRequest = new OMKeyCreateRequest(omRequest); + omKeyCreateRequest = getOMKeyCreateRequest(omRequest); OMClientResponse omClientResponse = omKeyCreateRequest.validateAndUpdateCache(ozoneManager, @@ -483,7 +483,7 @@ public class TestOMKeyCreateRequest extends TestOMKeyRequest { checkCreatedPaths(omKeyCreateRequest, omRequest, keyName); } - private void checkCreatedPaths(OMKeyCreateRequest omKeyCreateRequest, + protected void checkCreatedPaths(OMKeyCreateRequest omKeyCreateRequest, OMRequest omRequest, String keyName) throws Exception { keyName = omKeyCreateRequest.validateAndNormalizeKey(true, keyName); // Check intermediate directories created or not. @@ -497,9 +497,7 @@ public class TestOMKeyCreateRequest extends TestOMKeyRequest { Assert.assertNotNull(omKeyInfo); } - - - private void checkIntermediatePaths(Path keyPath) throws Exception { + protected long checkIntermediatePaths(Path keyPath) throws Exception { // Check intermediate paths are created keyPath = keyPath.getParent(); while(keyPath != null) { @@ -508,6 +506,15 @@ public class TestOMKeyCreateRequest extends TestOMKeyRequest { keyPath.toString()))); keyPath = keyPath.getParent(); } + return -1; + } + + protected String getOpenKey(long id) throws IOException { + return omMetadataManager.getOpenKey(volumeName, bucketName, + keyName, id); } + protected OMKeyCreateRequest getOMKeyCreateRequest(OMRequest omRequest) { + return new OMKeyCreateRequest(omRequest); + } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequestV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequestV1.java new file mode 100644 index 0000000..83c640d --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequestV1.java @@ -0,0 +1,129 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.request.key; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; +import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; +import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.util.Time; +import org.jetbrains.annotations.NotNull; +import org.junit.Assert; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Iterator; + +/** + * Tests OMCreateKeyRequestV1 class. + */ +public class TestOMKeyCreateRequestV1 extends TestOMKeyCreateRequest { + + @NotNull + @Override + protected OzoneConfiguration getOzoneConfiguration() { + OzoneConfiguration config = super.getOzoneConfiguration(); + config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); + // omLayoutVersionV1 flag will be set while invoking OzoneManager#start() + // and its not invoked in this test. Hence it is explicitly setting + // this configuration to populate prefix tables. + OzoneManagerRatisUtils.setOmLayoutVersionV1(true); + return config; + } + + protected void addToKeyTable(String keyName) throws Exception { + Path keyPath = Paths.get(keyName); + long parentId = checkIntermediatePaths(keyPath); + String fileName = OzoneFSUtils.getFileName(keyName); + OmKeyInfo omKeyInfo = + TestOMRequestUtils.createOmKeyInfo(volumeName, bucketName, fileName, + HddsProtos.ReplicationType.RATIS, + HddsProtos.ReplicationFactor.ONE, + parentId + 1, + parentId, 100, Time.now()); + TestOMRequestUtils.addFileToKeyTable(false, false, + fileName, omKeyInfo, -1, 50, omMetadataManager); + } + + protected void checkCreatedPaths(OMKeyCreateRequest omKeyCreateRequest, + OMRequest omRequest, String keyName) throws Exception { + keyName = omKeyCreateRequest.validateAndNormalizeKey(true, keyName); + // Check intermediate directories created or not. + Path keyPath = Paths.get(keyName); + long parentID = checkIntermediatePaths(keyPath); + + // Check open key entry + String fileName = keyPath.getFileName().toString(); + String openKey = omMetadataManager.getOpenFileName(parentID, fileName, + omRequest.getCreateKeyRequest().getClientID()); + OmKeyInfo omKeyInfo = omMetadataManager.getOpenKeyTable().get(openKey); + Assert.assertNotNull(omKeyInfo); + } + + protected long checkIntermediatePaths(Path keyPath) throws Exception { + // Check intermediate paths are created + keyPath = keyPath.getParent(); // skip the file name + String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName); + OmBucketInfo omBucketInfo = + omMetadataManager.getBucketTable().get(bucketKey); + long lastKnownParentId = omBucketInfo.getObjectID(); + + Iterator<Path> elements = keyPath.iterator(); + StringBuilder fullKeyPath = new StringBuilder(bucketKey); + while (elements.hasNext()) { + String fileName = elements.next().toString(); + fullKeyPath.append(OzoneConsts.OM_KEY_PREFIX); + fullKeyPath.append(fileName); + String dbNodeName = omMetadataManager.getOzonePathKey( + lastKnownParentId, fileName); + OmDirectoryInfo omDirInfo = omMetadataManager.getDirectoryTable(). + get(dbNodeName); + + Assert.assertNotNull("Parent key path:" + fullKeyPath + + " doesn't exist", omDirInfo); + lastKnownParentId = omDirInfo.getObjectID(); + } + + return lastKnownParentId; + } + + protected String getOpenKey(long id) throws IOException { + String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName); + OmBucketInfo omBucketInfo = + omMetadataManager.getBucketTable().get(bucketKey); + if (omBucketInfo != null) { + return omMetadataManager.getOpenFileName(omBucketInfo.getObjectID(), + keyName, id); + } else { + return omMetadataManager.getOpenFileName(1000, keyName, id); + } + } + + protected OMKeyCreateRequest getOMKeyCreateRequest(OMRequest omRequest) { + return new OMKeyCreateRequestV1(omRequest); + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponseV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponseV1.java index e105a37..92b3efe 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponseV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponseV1.java @@ -24,7 +24,6 @@ import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; @@ -64,8 +63,8 @@ public class TestOMAllocateBlockResponseV1 @NotNull protected OMAllocateBlockResponse getOmAllocateBlockResponse( - OmKeyInfo omKeyInfo, OmVolumeArgs omVolumeArgs, - OmBucketInfo omBucketInfo, OMResponse omResponse) { + OmKeyInfo omKeyInfo, OmBucketInfo omBucketInfo, + OMResponse omResponse) { return new OMAllocateBlockResponseV1(omResponse, omKeyInfo, clientID, omBucketInfo); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponseV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponseV1.java index 1e59ce8..4d68a4b 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponseV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponseV1.java @@ -42,8 +42,7 @@ public class TestOMKeyCommitResponseV1 extends TestOMKeyCommitResponse { OzoneManagerProtocolProtos.OMResponse omResponse, String openKey, String ozoneKey) { Assert.assertNotNull(omBucketInfo); - return new OMKeyCommitResponseV1( - omResponse, omKeyInfo, ozoneKey, openKey, omVolumeArgs, + return new OMKeyCommitResponseV1(omResponse, omKeyInfo, ozoneKey, openKey, omBucketInfo); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponseV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCreateResponseV1.java similarity index 60% copy from hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponseV1.java copy to hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCreateResponseV1.java index 1e59ce8..e51a06b 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponseV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCreateResponseV1.java @@ -19,61 +19,31 @@ package org.apache.hadoop.ozone.om.response.key; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; -import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; import org.apache.hadoop.util.Time; import org.jetbrains.annotations.NotNull; import org.junit.Assert; /** - * Tests OMKeyCommitResponse layout version V1. + * Tests OMKeyCreateResponseV1. */ -public class TestOMKeyCommitResponseV1 extends TestOMKeyCommitResponse { - - @NotNull - protected OMKeyCommitResponse getOmKeyCommitResponse( - OmVolumeArgs omVolumeArgs, OmKeyInfo omKeyInfo, - OzoneManagerProtocolProtos.OMResponse omResponse, String openKey, - String ozoneKey) { - Assert.assertNotNull(omBucketInfo); - return new OMKeyCommitResponseV1( - omResponse, omKeyInfo, ozoneKey, openKey, omVolumeArgs, - omBucketInfo); - } +public class TestOMKeyCreateResponseV1 extends TestOMKeyCreateResponse { @NotNull @Override - protected OmKeyInfo getOmKeyInfo() { - Assert.assertNotNull(omBucketInfo); - return TestOMRequestUtils.createOmKeyInfo(volumeName, - omBucketInfo.getBucketName(), keyName, replicationType, - replicationFactor, - omBucketInfo.getObjectID() + 1, - omBucketInfo.getObjectID(), 100, Time.now()); - } - - @NotNull - @Override - protected void addKeyToOpenKeyTable() throws Exception { - Assert.assertNotNull(omBucketInfo); - long parentID = omBucketInfo.getObjectID(); - long objectId = parentID + 10; - - OmKeyInfo omKeyInfoV1 = - TestOMRequestUtils.createOmKeyInfo(volumeName, bucketName, keyName, - HddsProtos.ReplicationType.RATIS, - HddsProtos.ReplicationFactor.ONE, objectId, parentID, 100, - Time.now()); - - String fileName = OzoneFSUtils.getFileName(keyName); - TestOMRequestUtils.addFileToKeyTable(true, false, - fileName, omKeyInfoV1, clientID, txnLogId, omMetadataManager); + protected OzoneConfiguration getOzoneConfiguration() { + OzoneConfiguration config = super.getOzoneConfiguration(); + config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); + // omLayoutVersionV1 flag will be set while invoking OzoneManager#start() + // and its not invoked in this test. Hence it is explicitly setting + // this configuration to populate prefix tables. + OzoneManagerRatisUtils.setOmLayoutVersionV1(true); + return config; } @NotNull @@ -86,21 +56,20 @@ public class TestOMKeyCommitResponseV1 extends TestOMKeyCommitResponse { @NotNull @Override - protected String getOzoneKey() { + protected OmKeyInfo getOmKeyInfo() { Assert.assertNotNull(omBucketInfo); - return omMetadataManager.getOzonePathKey(omBucketInfo.getObjectID(), - keyName); + return TestOMRequestUtils.createOmKeyInfo(volumeName, + omBucketInfo.getBucketName(), keyName, replicationType, + replicationFactor, + omBucketInfo.getObjectID() + 1, + omBucketInfo.getObjectID(), 100, Time.now()); } @NotNull - @Override - protected OzoneConfiguration getOzoneConfiguration() { - OzoneConfiguration config = super.getOzoneConfiguration(); - config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); - // omLayoutVersionV1 flag will be set while invoking OzoneManager#start() - // and its not invoked in this test. Hence it is explicitly setting - // this configuration to populate prefix tables. - OzoneManagerRatisUtils.setOmLayoutVersionV1(true); - return config; + protected OMKeyCreateResponse getOmKeyCreateResponse(OmKeyInfo keyInfo, + OmBucketInfo bucketInfo, OMResponse response) { + + return new OMKeyCreateResponseV1(response, keyInfo, null, clientID, + bucketInfo); } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
