sumitagrawl commented on code in PR #5517:
URL: https://github.com/apache/ozone/pull/5517#discussion_r1448424630
##########
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/LegacyBucketHandler.java:
##########
@@ -267,6 +344,35 @@ public long handleDirectKeys(long parentId, boolean
withReplica,
return keyDataSizeWithReplica;
}
+ /**
+ * This method handles disk usage calculation for legacy buckets.
+ * @param keyInfo the key info
+ * @param withReplica if withReplica is enabled, set sizeWithReplica
+ * @param listFile if listFile is enabled, append key DU as a subpath
+ */
+ private DUResponse.DiskUsage createDiskUsage(OmKeyInfo keyInfo,
+ boolean withReplica,
+ boolean listFile,
+ List<DUResponse.DiskUsage> duData) {
+ DUResponse.DiskUsage diskUsage = new DUResponse.DiskUsage();
+ String objectName = keyInfo.getKeyName();
+ diskUsage.setSubpath(objectName);
Review Comment:
earlier used buildSubPath, plz check using objectName do same logic
##########
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskWithLegacy.java:
##########
@@ -300,17 +321,53 @@ private void setKeyParentID(OmKeyInfo keyInfo) throws
IOException {
"NSSummaryTaskWithLegacy is null");
}
} else {
- String bucketKey = getReconOMMetadataManager()
- .getBucketKey(keyInfo.getVolumeName(), keyInfo.getBucketName());
- OmBucketInfo parentBucketInfo =
- getReconOMMetadataManager().getBucketTable().getSkipCache(bucketKey);
+ setParentBucketId(keyInfo);
+ }
+ }
- if (parentBucketInfo != null) {
- keyInfo.setParentObjectID(parentBucketInfo.getObjectID());
- } else {
- throw new IOException("ParentKeyInfo for " +
- "NSSummaryTaskWithLegacy is null");
- }
+ /**
+ * Set the parent object ID for a bucket.
+ *@paramkeyInfo
+ *@throwsIOException
+ */
+ private void setParentBucketId(OmKeyInfo keyInfo)
+ throws IOException {
+ String bucketKey = getReconOMMetadataManager()
+ .getBucketKey(keyInfo.getVolumeName(), keyInfo.getBucketName());
+ OmBucketInfo parentBucketInfo =
+ getReconOMMetadataManager().getBucketTable().getSkipCache(bucketKey);
+
+ if (parentBucketInfo != null) {
+ keyInfo.setParentObjectID(parentBucketInfo.getObjectID());
+ } else {
+ throw new IOException("ParentKeyInfo for " +
+ "NSSummaryTaskWithLegacy is null");
+ }
+ }
+
+ /**
+ * Check if the bucket layout is LEGACY.
+ * @param metadataManager
+ * @param keyInfo
+ * @return
+ */
+ private boolean isBucketLayoutValid(ReconOMMetadataManager metadataManager,
+ OmKeyInfo keyInfo)
+ throws IOException {
+ String volumeName = keyInfo.getVolumeName();
+ String bucketName = keyInfo.getBucketName();
+ String bucketDBKey = metadataManager.getBucketKey(volumeName, bucketName);
+ OmBucketInfo omBucketInfo =
+ metadataManager.getBucketTable().getSkipCache(bucketDBKey);
+
+ if (omBucketInfo.getBucketLayout() != BUCKET_LAYOUT) {
Review Comment:
rename BUCKET_LAYOUT as LEGACY_ BUCKET_LAYOUT
##########
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/LegacyBucketHandler.java:
##########
@@ -212,13 +220,93 @@ public long handleDirectKeys(long parentId, boolean
withReplica,
return 0;
}
+ if (enableFileSystemPaths) {
+ keyDataSizeWithReplica +=
+ handleDirectKeysForFSOLayout(parentId, withReplica, listFile, duData,
+ keyTable, seekPrefix, nsSummary);
+ } else {
+ keyDataSizeWithReplica +=
+ handleDirectKeysForOBSLayout(withReplica, listFile, duData, keyTable,
+ seekPrefix);
+ }
+
+ return keyDataSizeWithReplica;
+ }
+
+ /**
+ * This method handles disk usage of direct keys for OBS layout.
+ * @param withReplica if withReplica is enabled, set sizeWithReplica
+ * @param listFile if listFile is enabled, append key DU as a subpath
+ * @param duData the current DU data
+ * @param keyTable the key table
+ * @param seekPrefix the seek prefix used to position the iterator
+ * @return the total DU of all direct keys
+ * @throws IOException
+ */
+ public long handleDirectKeysForOBSLayout(boolean withReplica,
Review Comment:
this can also rename similarly as **LegacyOBS
##########
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/LegacyBucketHandler.java:
##########
@@ -212,13 +220,93 @@ public long handleDirectKeys(long parentId, boolean
withReplica,
return 0;
}
+ if (enableFileSystemPaths) {
+ keyDataSizeWithReplica +=
+ handleDirectKeysForFSOLayout(parentId, withReplica, listFile, duData,
+ keyTable, seekPrefix, nsSummary);
+ } else {
+ keyDataSizeWithReplica +=
+ handleDirectKeysForOBSLayout(withReplica, listFile, duData, keyTable,
+ seekPrefix);
+ }
+
+ return keyDataSizeWithReplica;
+ }
+
+ /**
+ * This method handles disk usage of direct keys for OBS layout.
+ * @param withReplica if withReplica is enabled, set sizeWithReplica
+ * @param listFile if listFile is enabled, append key DU as a subpath
+ * @param duData the current DU data
+ * @param keyTable the key table
+ * @param seekPrefix the seek prefix used to position the iterator
+ * @return the total DU of all direct keys
+ * @throws IOException
+ */
+ public long handleDirectKeysForOBSLayout(boolean withReplica,
+ boolean listFile,
+ List<DUResponse.DiskUsage> duData,
+ Table<String, OmKeyInfo> keyTable,
+ String seekPrefix)
+ throws IOException {
+ long keyDataSizeWithReplica = 0L;
+
+
+ try (
+ TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
iterator = keyTable.iterator()) {
+ iterator.seek(seekPrefix);
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, OmKeyInfo> kv = iterator.next();
+ String dbKey = kv.getKey();
+
+ if (!dbKey.startsWith(seekPrefix)) {
+ break;
+ }
+
+ OmKeyInfo keyInfo = kv.getValue();
+ if (keyInfo != null) {
+ createDiskUsage(keyInfo, withReplica, listFile, duData);
+ if (withReplica) {
+ long keyDU = keyInfo.getReplicatedSize();
+ keyDataSizeWithReplica += keyDU;
+ }
+ }
+ }
+ }
+
+ return keyDataSizeWithReplica;
+ }
+
+ /**
+ * This method handles disk usage of direct keys for OBS layout.
+ * @param parentId parent directory/bucket
+ * @param withReplica if withReplica is enabled, set sizeWithReplica
+ * @param listFile if listFile is enabled, append key DU as a subpath
+ * @param duData the current DU data
+ * @param keyTable the key table
+ * @param seekPrefix the seek prefix used to position the iterator
+ * @param nsSummary of the parent directory/bucket
+ * @return the total DU of all direct keys
+ * @throws IOException
+ */
+ public long handleDirectKeysForFSOLayout(long parentId, boolean withReplica,
+ boolean listFile,
+ List<DUResponse.DiskUsage> duData,
+ Table<String, OmKeyInfo> keyTable,
+ String seekPrefix,
+ NSSummary nsSummary)
+ throws IOException {
+ long keyDataSizeWithReplica = 0L;
+
if (omBucketInfo.getObjectID() != parentId) {
String dirName = nsSummary.getDirName();
seekPrefix += dirName;
}
String[] seekKeys = seekPrefix.split(OM_KEY_PREFIX);
- try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
- iterator = keyTable.iterator()) {
+ try (
+ TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
Review Comment:
We are getting all keys in directory, right? not a recursive listing?
##########
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/LegacyBucketHandler.java:
##########
@@ -212,13 +220,93 @@ public long handleDirectKeys(long parentId, boolean
withReplica,
return 0;
}
+ if (enableFileSystemPaths) {
+ keyDataSizeWithReplica +=
+ handleDirectKeysForFSOLayout(parentId, withReplica, listFile, duData,
+ keyTable, seekPrefix, nsSummary);
+ } else {
+ keyDataSizeWithReplica +=
+ handleDirectKeysForOBSLayout(withReplica, listFile, duData, keyTable,
+ seekPrefix);
+ }
+
+ return keyDataSizeWithReplica;
+ }
+
+ /**
+ * This method handles disk usage of direct keys for OBS layout.
+ * @param withReplica if withReplica is enabled, set sizeWithReplica
+ * @param listFile if listFile is enabled, append key DU as a subpath
+ * @param duData the current DU data
+ * @param keyTable the key table
+ * @param seekPrefix the seek prefix used to position the iterator
+ * @return the total DU of all direct keys
+ * @throws IOException
+ */
+ public long handleDirectKeysForOBSLayout(boolean withReplica,
+ boolean listFile,
+ List<DUResponse.DiskUsage> duData,
+ Table<String, OmKeyInfo> keyTable,
+ String seekPrefix)
+ throws IOException {
+ long keyDataSizeWithReplica = 0L;
+
+
+ try (
+ TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
iterator = keyTable.iterator()) {
+ iterator.seek(seekPrefix);
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, OmKeyInfo> kv = iterator.next();
+ String dbKey = kv.getKey();
+
+ if (!dbKey.startsWith(seekPrefix)) {
+ break;
+ }
+
+ OmKeyInfo keyInfo = kv.getValue();
+ if (keyInfo != null) {
+ createDiskUsage(keyInfo, withReplica, listFile, duData);
+ if (withReplica) {
+ long keyDU = keyInfo.getReplicatedSize();
+ keyDataSizeWithReplica += keyDU;
+ }
+ }
+ }
+ }
+
+ return keyDataSizeWithReplica;
+ }
+
+ /**
+ * This method handles disk usage of direct keys for OBS layout.
+ * @param parentId parent directory/bucket
+ * @param withReplica if withReplica is enabled, set sizeWithReplica
+ * @param listFile if listFile is enabled, append key DU as a subpath
+ * @param duData the current DU data
+ * @param keyTable the key table
+ * @param seekPrefix the seek prefix used to position the iterator
+ * @param nsSummary of the parent directory/bucket
+ * @return the total DU of all direct keys
+ * @throws IOException
+ */
+ public long handleDirectKeysForFSOLayout(long parentId, boolean withReplica,
Review Comment:
This is not FSO, rename handleDirectKeysForLegacyFS, i.e. legacy file system
and update comment also
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
