ashishkumar50 commented on code in PR #7455:
URL: https://github.com/apache/ozone/pull/7455#discussion_r1858011118
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java:
##########
@@ -470,12 +490,209 @@ protected static List<OzoneAcl> getAclsForDir(KeyArgs
keyArgs,
OzoneAclUtil.inheritDefaultAcls(acls, bucketInfo.getAcls(), DEFAULT);
}
- // add itself acls
+ // add acls from clients
acls.addAll(OzoneAclUtil.fromProtobuf(keyArgs.getAclsList()));
-
+ acls = acls.stream().distinct().collect(Collectors.toList());
return acls;
}
+ static long getMaxNumOfRecursiveDirs() {
+ return MAX_NUM_OF_RECURSIVE_DIRS;
+ }
+
+ /**
+ * Construct OmDirectoryInfo for every parent directory in missing list.
+ *
+ * @param keyArgs key arguments
+ * @param pathInfo list of parent directories to be created and its ACLs
+ * @param trxnLogIndex transaction log index id
+ * @return list of missing parent directories
+ * @throws IOException DB failure
+ */
+ protected List<OmDirectoryInfo> getAllMissingParentDirInfo(
+ OzoneManager ozoneManager, KeyArgs keyArgs, OmBucketInfo bucketInfo,
+ OMFileRequest.OMPathInfoWithFSO pathInfo, long trxnLogIndex)
+ throws IOException {
+ List<OmDirectoryInfo> missingParentInfos = new ArrayList<>();
+
+ // The base id is left shifted by 8 bits for creating space to
+ // create (2^8 - 1) object ids in every request.
+ // maxObjId represents the largest object id allocation possible inside
+ // the transaction.
+ long baseObjId = ozoneManager.getObjectIdFromTxId(trxnLogIndex);
+ long maxObjId = baseObjId + getMaxNumOfRecursiveDirs();
+ long objectCount = 1;
+
+ String volumeName = keyArgs.getVolumeName();
+ String bucketName = keyArgs.getBucketName();
+ String keyName = keyArgs.getKeyName();
+
+ long lastKnownParentId = pathInfo.getLastKnownParentId();
+ List<String> missingParents = pathInfo.getMissingParents();
+ for (String missingKey : missingParents) {
+ long nextObjId = baseObjId + objectCount;
+ if (nextObjId > maxObjId) {
+ throw new OMException("Too many directories in path. Exceeds limit of "
+ + getMaxNumOfRecursiveDirs() + ". Unable to create directory: "
+ + keyName + " in volume/bucket: " + volumeName + "/" + bucketName,
+ INVALID_KEY_NAME);
+ }
+
+ LOG.debug("missing parent {} getting added to DirectoryTable",
+ missingKey);
+ OmDirectoryInfo dirInfo = createDirectoryInfoWithACL(missingKey,
+ keyArgs, nextObjId, lastKnownParentId, trxnLogIndex,
+ bucketInfo, pathInfo, ozoneManager.getConfiguration());
+ objectCount++;
+
+ missingParentInfos.add(dirInfo);
+
+ // updating id for the next sub-dir
+ lastKnownParentId = nextObjId;
+ }
+ pathInfo.setLastKnownParentId(lastKnownParentId);
+ pathInfo.setLeafNodeObjectId(baseObjId + objectCount);
+ return missingParentInfos;
+ }
+
+ /**
+ * Construct OmKeyInfo for every parent directory in missing list.
+ * @param ozoneManager
+ * @param keyArgs
+ * @param missingParents list of parent directories to be created
+ * @param bucketInfo
+ * @param omPathInfo
+ * @param trxnLogIndex
+ * @return {@code List<OmKeyInfo>}
+ * @throws IOException
+ */
+ protected List<OmKeyInfo> getAllParentInfo(OzoneManager ozoneManager,
+ KeyArgs keyArgs, List<String> missingParents, OmBucketInfo bucketInfo,
+ OMFileRequest.OMPathInfo omPathInfo, long trxnLogIndex)
+ throws IOException {
+ List<OmKeyInfo> missingParentInfos = new ArrayList<>();
+
+ // The base id is left shifted by 8 bits for creating space to
+ // create (2^8 - 1) object ids in every request.
+ // maxObjId represents the largest object id allocation possible inside
+ // the transaction.
+ long baseObjId = ozoneManager.getObjectIdFromTxId(trxnLogIndex);
+ long maxObjId = baseObjId + MAX_NUM_OF_RECURSIVE_DIRS;
+ long objectCount = 1; // baseObjID is used by the leaf directory
+
+ String volumeName = keyArgs.getVolumeName();
+ String bucketName = keyArgs.getBucketName();
+ String keyName = keyArgs.getKeyName();
+
+ for (String missingKey : missingParents) {
+ long nextObjId = baseObjId + objectCount;
+ if (nextObjId > maxObjId) {
+ throw new OMException("Too many directories in path. Exceeds limit of "
+ + MAX_NUM_OF_RECURSIVE_DIRS + ". Unable to create directory: "
+ + keyName + " in volume/bucket: " + volumeName + "/" + bucketName,
+ INVALID_KEY_NAME);
+ }
+
+ LOG.debug("missing parent {} getting added to KeyTable", missingKey);
+
+ OmKeyInfo parentKeyInfo =
+ createDirectoryKeyInfoWithNoACL(missingKey, keyArgs, nextObjId,
+ bucketInfo, omPathInfo, trxnLogIndex,
+ ozoneManager.getDefaultReplicationConfig(),
ozoneManager.getConfiguration());
+ objectCount++;
+
+ missingParentInfos.add(parentKeyInfo);
+ }
+
+ return missingParentInfos;
+ }
+
+ /**
+ * Fill in a DirectoryInfo for a new directory entry in OM database.
+ * @param dirName
+ * @param keyArgs
+ * @param objectId
+ * @param parentObjectId
+ * @param bucketInfo
+ * @param omPathInfo
Review Comment:
`* @param config`
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java:
##########
@@ -470,12 +490,209 @@ protected static List<OzoneAcl> getAclsForDir(KeyArgs
keyArgs,
OzoneAclUtil.inheritDefaultAcls(acls, bucketInfo.getAcls(), DEFAULT);
}
- // add itself acls
+ // add acls from clients
acls.addAll(OzoneAclUtil.fromProtobuf(keyArgs.getAclsList()));
-
+ acls = acls.stream().distinct().collect(Collectors.toList());
return acls;
}
+ static long getMaxNumOfRecursiveDirs() {
+ return MAX_NUM_OF_RECURSIVE_DIRS;
+ }
+
+ /**
+ * Construct OmDirectoryInfo for every parent directory in missing list.
+ *
+ * @param keyArgs key arguments
+ * @param pathInfo list of parent directories to be created and its ACLs
+ * @param trxnLogIndex transaction log index id
+ * @return list of missing parent directories
+ * @throws IOException DB failure
+ */
+ protected List<OmDirectoryInfo> getAllMissingParentDirInfo(
+ OzoneManager ozoneManager, KeyArgs keyArgs, OmBucketInfo bucketInfo,
+ OMFileRequest.OMPathInfoWithFSO pathInfo, long trxnLogIndex)
+ throws IOException {
+ List<OmDirectoryInfo> missingParentInfos = new ArrayList<>();
+
+ // The base id is left shifted by 8 bits for creating space to
+ // create (2^8 - 1) object ids in every request.
+ // maxObjId represents the largest object id allocation possible inside
+ // the transaction.
+ long baseObjId = ozoneManager.getObjectIdFromTxId(trxnLogIndex);
+ long maxObjId = baseObjId + getMaxNumOfRecursiveDirs();
+ long objectCount = 1;
+
+ String volumeName = keyArgs.getVolumeName();
+ String bucketName = keyArgs.getBucketName();
+ String keyName = keyArgs.getKeyName();
+
+ long lastKnownParentId = pathInfo.getLastKnownParentId();
+ List<String> missingParents = pathInfo.getMissingParents();
+ for (String missingKey : missingParents) {
+ long nextObjId = baseObjId + objectCount;
+ if (nextObjId > maxObjId) {
+ throw new OMException("Too many directories in path. Exceeds limit of "
+ + getMaxNumOfRecursiveDirs() + ". Unable to create directory: "
+ + keyName + " in volume/bucket: " + volumeName + "/" + bucketName,
+ INVALID_KEY_NAME);
+ }
+
+ LOG.debug("missing parent {} getting added to DirectoryTable",
+ missingKey);
+ OmDirectoryInfo dirInfo = createDirectoryInfoWithACL(missingKey,
+ keyArgs, nextObjId, lastKnownParentId, trxnLogIndex,
+ bucketInfo, pathInfo, ozoneManager.getConfiguration());
+ objectCount++;
+
+ missingParentInfos.add(dirInfo);
+
+ // updating id for the next sub-dir
+ lastKnownParentId = nextObjId;
+ }
+ pathInfo.setLastKnownParentId(lastKnownParentId);
+ pathInfo.setLeafNodeObjectId(baseObjId + objectCount);
+ return missingParentInfos;
+ }
+
+ /**
+ * Construct OmKeyInfo for every parent directory in missing list.
+ * @param ozoneManager
+ * @param keyArgs
+ * @param missingParents list of parent directories to be created
+ * @param bucketInfo
+ * @param omPathInfo
+ * @param trxnLogIndex
+ * @return {@code List<OmKeyInfo>}
+ * @throws IOException
+ */
+ protected List<OmKeyInfo> getAllParentInfo(OzoneManager ozoneManager,
+ KeyArgs keyArgs, List<String> missingParents, OmBucketInfo bucketInfo,
+ OMFileRequest.OMPathInfo omPathInfo, long trxnLogIndex)
+ throws IOException {
+ List<OmKeyInfo> missingParentInfos = new ArrayList<>();
+
+ // The base id is left shifted by 8 bits for creating space to
+ // create (2^8 - 1) object ids in every request.
+ // maxObjId represents the largest object id allocation possible inside
+ // the transaction.
+ long baseObjId = ozoneManager.getObjectIdFromTxId(trxnLogIndex);
+ long maxObjId = baseObjId + MAX_NUM_OF_RECURSIVE_DIRS;
+ long objectCount = 1; // baseObjID is used by the leaf directory
+
+ String volumeName = keyArgs.getVolumeName();
+ String bucketName = keyArgs.getBucketName();
+ String keyName = keyArgs.getKeyName();
+
+ for (String missingKey : missingParents) {
+ long nextObjId = baseObjId + objectCount;
+ if (nextObjId > maxObjId) {
+ throw new OMException("Too many directories in path. Exceeds limit of "
+ + MAX_NUM_OF_RECURSIVE_DIRS + ". Unable to create directory: "
+ + keyName + " in volume/bucket: " + volumeName + "/" + bucketName,
+ INVALID_KEY_NAME);
+ }
+
+ LOG.debug("missing parent {} getting added to KeyTable", missingKey);
+
+ OmKeyInfo parentKeyInfo =
+ createDirectoryKeyInfoWithNoACL(missingKey, keyArgs, nextObjId,
+ bucketInfo, omPathInfo, trxnLogIndex,
+ ozoneManager.getDefaultReplicationConfig(),
ozoneManager.getConfiguration());
+ objectCount++;
+
+ missingParentInfos.add(parentKeyInfo);
+ }
+
+ return missingParentInfos;
+ }
+
+ /**
+ * Fill in a DirectoryInfo for a new directory entry in OM database.
+ * @param dirName
+ * @param keyArgs
+ * @param objectId
+ * @param parentObjectId
+ * @param bucketInfo
+ * @param omPathInfo
+ * @return the OmDirectoryInfo structure
+ */
+ @SuppressWarnings("parameternumber")
+ protected OmDirectoryInfo createDirectoryInfoWithACL(
+ String dirName, KeyArgs keyArgs, long objectId,
+ long parentObjectId, long transactionIndex,
+ OmBucketInfo bucketInfo, OMFileRequest.OMPathInfo omPathInfo,
+ OzoneConfiguration config) throws OMException {
+ return OmDirectoryInfo.newBuilder()
+ .setName(dirName)
+ .setOwner(keyArgs.getOwnerName())
+ .setCreationTime(keyArgs.getModificationTime())
+ .setModificationTime(keyArgs.getModificationTime())
+ .setObjectID(objectId)
+ .setUpdateID(transactionIndex)
+ .setParentObjectID(parentObjectId).setAcls(getAclsForDir(keyArgs,
bucketInfo, omPathInfo, config))
+ .build();
+ }
+
+ /**
+ * fill in a KeyInfo for a new directory entry in OM database.
+ * without initializing ACLs from the KeyArgs - used for intermediate
+ * directories which get created internally/recursively during file
+ * and directory create.
+ * @param keyName
+ * @param keyArgs
+ * @param objectId
+ * @param bucketInfo
+ * @param omPathInfo
+ * @param transactionIndex
+ * @param serverDefaultReplConfig
+ * @return the OmKeyInfo structure
+ */
+ @SuppressWarnings("parameternumber")
+ protected OmKeyInfo createDirectoryKeyInfoWithNoACL(String keyName,
+ KeyArgs keyArgs, long objectId, OmBucketInfo bucketInfo,
+ OMFileRequest.OMPathInfo omPathInfo, long transactionIndex,
+ ReplicationConfig serverDefaultReplConfig, OzoneConfiguration config)
throws OMException {
+ return dirKeyInfoBuilderNoACL(keyName, keyArgs, objectId,
+ serverDefaultReplConfig)
+ .setAcls(getAclsForDir(keyArgs, bucketInfo, omPathInfo, config))
Review Comment:
Do we need `setAcls` here?
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java:
##########
@@ -470,12 +490,209 @@ protected static List<OzoneAcl> getAclsForDir(KeyArgs
keyArgs,
OzoneAclUtil.inheritDefaultAcls(acls, bucketInfo.getAcls(), DEFAULT);
}
- // add itself acls
+ // add acls from clients
acls.addAll(OzoneAclUtil.fromProtobuf(keyArgs.getAclsList()));
-
+ acls = acls.stream().distinct().collect(Collectors.toList());
return acls;
}
+ static long getMaxNumOfRecursiveDirs() {
+ return MAX_NUM_OF_RECURSIVE_DIRS;
+ }
Review Comment:
`getMaxNumOfRecursiveDirs()` is not required, `MAX_NUM_OF_RECURSIVE_DIRS`
can be accessed directly now.
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java:
##########
@@ -470,12 +490,209 @@ protected static List<OzoneAcl> getAclsForDir(KeyArgs
keyArgs,
OzoneAclUtil.inheritDefaultAcls(acls, bucketInfo.getAcls(), DEFAULT);
}
- // add itself acls
+ // add acls from clients
acls.addAll(OzoneAclUtil.fromProtobuf(keyArgs.getAclsList()));
-
+ acls = acls.stream().distinct().collect(Collectors.toList());
return acls;
}
+ static long getMaxNumOfRecursiveDirs() {
+ return MAX_NUM_OF_RECURSIVE_DIRS;
+ }
+
+ /**
+ * Construct OmDirectoryInfo for every parent directory in missing list.
+ *
+ * @param keyArgs key arguments
+ * @param pathInfo list of parent directories to be created and its ACLs
+ * @param trxnLogIndex transaction log index id
+ * @return list of missing parent directories
+ * @throws IOException DB failure
+ */
+ protected List<OmDirectoryInfo> getAllMissingParentDirInfo(
+ OzoneManager ozoneManager, KeyArgs keyArgs, OmBucketInfo bucketInfo,
+ OMFileRequest.OMPathInfoWithFSO pathInfo, long trxnLogIndex)
+ throws IOException {
+ List<OmDirectoryInfo> missingParentInfos = new ArrayList<>();
+
+ // The base id is left shifted by 8 bits for creating space to
+ // create (2^8 - 1) object ids in every request.
+ // maxObjId represents the largest object id allocation possible inside
+ // the transaction.
+ long baseObjId = ozoneManager.getObjectIdFromTxId(trxnLogIndex);
+ long maxObjId = baseObjId + getMaxNumOfRecursiveDirs();
+ long objectCount = 1;
+
+ String volumeName = keyArgs.getVolumeName();
+ String bucketName = keyArgs.getBucketName();
+ String keyName = keyArgs.getKeyName();
+
+ long lastKnownParentId = pathInfo.getLastKnownParentId();
+ List<String> missingParents = pathInfo.getMissingParents();
+ for (String missingKey : missingParents) {
+ long nextObjId = baseObjId + objectCount;
+ if (nextObjId > maxObjId) {
+ throw new OMException("Too many directories in path. Exceeds limit of "
+ + getMaxNumOfRecursiveDirs() + ". Unable to create directory: "
+ + keyName + " in volume/bucket: " + volumeName + "/" + bucketName,
+ INVALID_KEY_NAME);
+ }
+
+ LOG.debug("missing parent {} getting added to DirectoryTable",
+ missingKey);
+ OmDirectoryInfo dirInfo = createDirectoryInfoWithACL(missingKey,
+ keyArgs, nextObjId, lastKnownParentId, trxnLogIndex,
+ bucketInfo, pathInfo, ozoneManager.getConfiguration());
+ objectCount++;
+
+ missingParentInfos.add(dirInfo);
+
+ // updating id for the next sub-dir
+ lastKnownParentId = nextObjId;
+ }
+ pathInfo.setLastKnownParentId(lastKnownParentId);
+ pathInfo.setLeafNodeObjectId(baseObjId + objectCount);
+ return missingParentInfos;
+ }
+
+ /**
+ * Construct OmKeyInfo for every parent directory in missing list.
+ * @param ozoneManager
+ * @param keyArgs
+ * @param missingParents list of parent directories to be created
+ * @param bucketInfo
+ * @param omPathInfo
+ * @param trxnLogIndex
+ * @return {@code List<OmKeyInfo>}
+ * @throws IOException
+ */
+ protected List<OmKeyInfo> getAllParentInfo(OzoneManager ozoneManager,
+ KeyArgs keyArgs, List<String> missingParents, OmBucketInfo bucketInfo,
+ OMFileRequest.OMPathInfo omPathInfo, long trxnLogIndex)
+ throws IOException {
+ List<OmKeyInfo> missingParentInfos = new ArrayList<>();
+
+ // The base id is left shifted by 8 bits for creating space to
+ // create (2^8 - 1) object ids in every request.
+ // maxObjId represents the largest object id allocation possible inside
+ // the transaction.
+ long baseObjId = ozoneManager.getObjectIdFromTxId(trxnLogIndex);
+ long maxObjId = baseObjId + MAX_NUM_OF_RECURSIVE_DIRS;
+ long objectCount = 1; // baseObjID is used by the leaf directory
+
+ String volumeName = keyArgs.getVolumeName();
+ String bucketName = keyArgs.getBucketName();
+ String keyName = keyArgs.getKeyName();
+
+ for (String missingKey : missingParents) {
+ long nextObjId = baseObjId + objectCount;
+ if (nextObjId > maxObjId) {
+ throw new OMException("Too many directories in path. Exceeds limit of "
+ + MAX_NUM_OF_RECURSIVE_DIRS + ". Unable to create directory: "
+ + keyName + " in volume/bucket: " + volumeName + "/" + bucketName,
+ INVALID_KEY_NAME);
+ }
+
+ LOG.debug("missing parent {} getting added to KeyTable", missingKey);
+
+ OmKeyInfo parentKeyInfo =
+ createDirectoryKeyInfoWithNoACL(missingKey, keyArgs, nextObjId,
+ bucketInfo, omPathInfo, trxnLogIndex,
+ ozoneManager.getDefaultReplicationConfig(),
ozoneManager.getConfiguration());
+ objectCount++;
+
+ missingParentInfos.add(parentKeyInfo);
+ }
+
+ return missingParentInfos;
+ }
+
+ /**
+ * Fill in a DirectoryInfo for a new directory entry in OM database.
+ * @param dirName
+ * @param keyArgs
+ * @param objectId
+ * @param parentObjectId
+ * @param bucketInfo
+ * @param omPathInfo
+ * @return the OmDirectoryInfo structure
+ */
+ @SuppressWarnings("parameternumber")
+ protected OmDirectoryInfo createDirectoryInfoWithACL(
+ String dirName, KeyArgs keyArgs, long objectId,
+ long parentObjectId, long transactionIndex,
+ OmBucketInfo bucketInfo, OMFileRequest.OMPathInfo omPathInfo,
+ OzoneConfiguration config) throws OMException {
+ return OmDirectoryInfo.newBuilder()
+ .setName(dirName)
+ .setOwner(keyArgs.getOwnerName())
+ .setCreationTime(keyArgs.getModificationTime())
+ .setModificationTime(keyArgs.getModificationTime())
+ .setObjectID(objectId)
+ .setUpdateID(transactionIndex)
+ .setParentObjectID(parentObjectId).setAcls(getAclsForDir(keyArgs,
bucketInfo, omPathInfo, config))
+ .build();
+ }
+
+ /**
+ * fill in a KeyInfo for a new directory entry in OM database.
+ * without initializing ACLs from the KeyArgs - used for intermediate
+ * directories which get created internally/recursively during file
+ * and directory create.
+ * @param keyName
+ * @param keyArgs
+ * @param objectId
+ * @param bucketInfo
+ * @param omPathInfo
+ * @param transactionIndex
+ * @param serverDefaultReplConfig
Review Comment:
`* @param config`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
