devmadhuu commented on code in PR #5517:
URL: https://github.com/apache/ozone/pull/5517#discussion_r1544348873
##########
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/EntityHandler.java:
##########
@@ -256,7 +280,52 @@ public static String[] parseRequestPath(String path) {
return names;
}
- private static String normalizePath(String path) {
+ /**
+ * Splits an object store path into volume, bucket, and key name components.
+ *
+ * This method parses a path of the format "/volumeName/bucketName/keyName",
+ * including paths with additional '/' characters within the key name. It's
+ * designed for object store paths where the first three '/' characters
+ * separate the root, volume and bucket names from the key name.
+ *
+ * @param path The object store path to parse, starting with a slash.
+ * @return A String array with three elements: volume name, bucket name, and
+ * key name, or {null} if the path format is invalid.
+ */
+ public static String[] parseObjectStorePath(String path) {
+ // Removing the leading slash for correct splitting
+ path = path.substring(1);
+
+ // Splitting the modified path by "/", limiting to 3 parts
+ String[] parts = path.split("/", 3);
Review Comment:
```suggestion
String[] parts = path.split(OM_KEY_PREFIX, 3);
```
##########
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java:
##########
@@ -743,6 +743,46 @@ public static String normalizeKey(String keyName,
return keyName;
}
+ /**
+ * Normalizes a given path up to the bucket level.
+ *
+ * This method takes a path as input and normalises uptil the bucket level.
+ * It handles empty, removes leading slashes, and splits the path into
+ * segments. It then extracts the volume and bucket names, forming a
+ * normalized path with a single slash. Finally, any remaining segments are
+ * joined as the key name, returning the complete standardized path.
+ *
+ * @param path The path string to be normalized.
+ * @return The normalized path string.
+ */
+ public static String normalizePathUptilBucket(String path) {
Review Comment:
Can we rename as `uptil` is not very familiar word to use in coding.
```suggestion
public static String normalizePathUptoBucket(String path) {
```
##########
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskWithLegacy.java:
##########
@@ -206,12 +131,118 @@ public boolean processWithLegacy(OMUpdateEventBatch
events) {
return true;
}
+ private void processWithFileSystemLayout(OmKeyInfo updatedKeyInfo,
+ OmKeyInfo oldKeyInfo,
+ OMDBUpdateEvent.OMDBUpdateAction
action,
+ Map<Long, NSSummary> nsSummaryMap)
+ throws IOException {
+ setKeyParentID(updatedKeyInfo);
+
+ if (!updatedKeyInfo.getKeyName().endsWith(OM_KEY_PREFIX)) {
+ switch (action) {
+ case PUT:
+ handlePutKeyEvent(updatedKeyInfo, nsSummaryMap);
+ break;
+
+ case DELETE:
+ handleDeleteKeyEvent(updatedKeyInfo, nsSummaryMap);
+ break;
+
+ case UPDATE:
+ if (oldKeyInfo != null) {
+ setKeyParentID(oldKeyInfo);
+ handleDeleteKeyEvent(oldKeyInfo, nsSummaryMap);
+ } else {
+ LOG.warn("Update event does not have the old keyInfo for {}.",
+ updatedKeyInfo.getKeyName());
+ }
+ handlePutKeyEvent(updatedKeyInfo, nsSummaryMap);
+ break;
+
+ default:
+ LOG.debug("Skipping DB update event fir Key: {}", action);
Review Comment:
```suggestion
if (LOG.isDebugEnabled()) {
LOG.debug("Skipping DB update event fir Key: {}", action);
}
```
##########
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskWithLegacy.java:
##########
@@ -206,12 +131,118 @@ public boolean processWithLegacy(OMUpdateEventBatch
events) {
return true;
}
+ private void processWithFileSystemLayout(OmKeyInfo updatedKeyInfo,
+ OmKeyInfo oldKeyInfo,
+ OMDBUpdateEvent.OMDBUpdateAction
action,
+ Map<Long, NSSummary> nsSummaryMap)
+ throws IOException {
+ setKeyParentID(updatedKeyInfo);
+
+ if (!updatedKeyInfo.getKeyName().endsWith(OM_KEY_PREFIX)) {
+ switch (action) {
+ case PUT:
+ handlePutKeyEvent(updatedKeyInfo, nsSummaryMap);
+ break;
+
+ case DELETE:
+ handleDeleteKeyEvent(updatedKeyInfo, nsSummaryMap);
+ break;
+
+ case UPDATE:
+ if (oldKeyInfo != null) {
+ setKeyParentID(oldKeyInfo);
+ handleDeleteKeyEvent(oldKeyInfo, nsSummaryMap);
+ } else {
+ LOG.warn("Update event does not have the old keyInfo for {}.",
+ updatedKeyInfo.getKeyName());
+ }
+ handlePutKeyEvent(updatedKeyInfo, nsSummaryMap);
+ break;
+
+ default:
+ LOG.debug("Skipping DB update event fir Key: {}", action);
Review Comment:
```suggestion
LOG.debug("Skipping DB update event for Key: {}", action);
```
##########
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/EntityHandler.java:
##########
@@ -256,7 +280,52 @@ public static String[] parseRequestPath(String path) {
return names;
}
- private static String normalizePath(String path) {
+ /**
+ * Splits an object store path into volume, bucket, and key name components.
+ *
+ * This method parses a path of the format "/volumeName/bucketName/keyName",
+ * including paths with additional '/' characters within the key name. It's
+ * designed for object store paths where the first three '/' characters
+ * separate the root, volume and bucket names from the key name.
+ *
+ * @param path The object store path to parse, starting with a slash.
+ * @return A String array with three elements: volume name, bucket name, and
+ * key name, or {null} if the path format is invalid.
+ */
+ public static String[] parseObjectStorePath(String path) {
+ // Removing the leading slash for correct splitting
+ path = path.substring(1);
+
+ // Splitting the modified path by "/", limiting to 3 parts
+ String[] parts = path.split("/", 3);
+
+ // Checking if we correctly obtained 3 parts after removing the leading
slash
+ if (parts.length <= 3) {
+ return parts;
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Normalizes a given path based on the specified bucket layout.
+ *
+ * This method adjusts the path according to the bucket layout.
+ * For {OBJECT_STORE Layout}, it normalizes the path up to the bucket level
+ * using OmUtils.normalizePathUptilBucket. For other layouts, it
Review Comment:
```suggestion
* using OmUtils.normalizePathUptoBucket. For other layouts, it
```
##########
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java:
##########
@@ -743,6 +743,46 @@ public static String normalizeKey(String keyName,
return keyName;
}
+ /**
+ * Normalizes a given path up to the bucket level.
+ *
+ * This method takes a path as input and normalises uptil the bucket level.
+ * It handles empty, removes leading slashes, and splits the path into
+ * segments. It then extracts the volume and bucket names, forming a
+ * normalized path with a single slash. Finally, any remaining segments are
+ * joined as the key name, returning the complete standardized path.
+ *
+ * @param path The path string to be normalized.
+ * @return The normalized path string.
+ */
+ public static String normalizePathUptilBucket(String path) {
+ if (path == null || path.isEmpty()) {
+ return "/"; // Handle empty path
Review Comment:
```suggestion
return OM_KEY_PREFIX; // Handle empty path
```
##########
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskWithLegacy.java:
##########
@@ -206,12 +131,118 @@ public boolean processWithLegacy(OMUpdateEventBatch
events) {
return true;
}
+ private void processWithFileSystemLayout(OmKeyInfo updatedKeyInfo,
+ OmKeyInfo oldKeyInfo,
+ OMDBUpdateEvent.OMDBUpdateAction
action,
+ Map<Long, NSSummary> nsSummaryMap)
+ throws IOException {
+ setKeyParentID(updatedKeyInfo);
+
+ if (!updatedKeyInfo.getKeyName().endsWith(OM_KEY_PREFIX)) {
+ switch (action) {
+ case PUT:
+ handlePutKeyEvent(updatedKeyInfo, nsSummaryMap);
+ break;
+
+ case DELETE:
+ handleDeleteKeyEvent(updatedKeyInfo, nsSummaryMap);
+ break;
+
+ case UPDATE:
+ if (oldKeyInfo != null) {
+ setKeyParentID(oldKeyInfo);
+ handleDeleteKeyEvent(oldKeyInfo, nsSummaryMap);
+ } else {
+ LOG.warn("Update event does not have the old keyInfo for {}.",
+ updatedKeyInfo.getKeyName());
+ }
+ handlePutKeyEvent(updatedKeyInfo, nsSummaryMap);
+ break;
+
+ default:
+ LOG.debug("Skipping DB update event fir Key: {}", action);
+ }
+ } else {
+ OmDirectoryInfo updatedDirectoryInfo = new OmDirectoryInfo.Builder()
+ .setName(updatedKeyInfo.getKeyName())
+ .setObjectID(updatedKeyInfo.getObjectID())
+ .setParentObjectID(updatedKeyInfo.getParentObjectID())
+ .build();
+
+ OmDirectoryInfo oldDirectoryInfo = null;
+
+ if (oldKeyInfo != null) {
+ oldDirectoryInfo =
+ new OmDirectoryInfo.Builder()
+ .setName(oldKeyInfo.getKeyName())
+ .setObjectID(oldKeyInfo.getObjectID())
+ .setParentObjectID(oldKeyInfo.getParentObjectID())
+ .build();
+ }
+
+ switch (action) {
+ case PUT:
+ handlePutDirEvent(updatedDirectoryInfo, nsSummaryMap);
+ break;
+
+ case DELETE:
+ handleDeleteDirEvent(updatedDirectoryInfo, nsSummaryMap);
+ break;
+
+ case UPDATE:
+ if (oldDirectoryInfo != null) {
+ handleDeleteDirEvent(oldDirectoryInfo, nsSummaryMap);
+ } else {
+ LOG.warn("Update event does not have the old dirInfo for {}.",
+ updatedKeyInfo.getKeyName());
+ }
+ handlePutDirEvent(updatedDirectoryInfo, nsSummaryMap);
+ break;
+
+ default:
+ LOG.debug("Skipping DB update event for Directory: {}", action);
+ }
+ }
+ }
+
+ private void processWithObjectStoreLayout(OmKeyInfo updatedKeyInfo,
+ OmKeyInfo oldKeyInfo,
+ OMDBUpdateEvent.OMDBUpdateAction
action,
+ Map<Long, NSSummary> nsSummaryMap)
+ throws IOException {
+ setParentBucketId(updatedKeyInfo);
+
+ switch (action) {
+ case PUT:
+ handlePutKeyEvent(updatedKeyInfo, nsSummaryMap);
+ break;
+
+ case DELETE:
+ handleDeleteKeyEvent(updatedKeyInfo, nsSummaryMap);
+ break;
+
+ case UPDATE:
+ if (oldKeyInfo != null) {
+ setParentBucketId(oldKeyInfo);
+ handleDeleteKeyEvent(oldKeyInfo, nsSummaryMap);
+ } else {
+ LOG.warn("Update event does not have the old keyInfo for {}.",
+ updatedKeyInfo.getKeyName());
+ }
+ handlePutKeyEvent(updatedKeyInfo, nsSummaryMap);
+ break;
+
+ default:
+ LOG.debug("Skipping DB update event for Key: {}", action);
Review Comment:
```suggestion
if(LOG.isDebugEnabled()) {
LOG.debug("Skipping DB update event for Key: {}", action);
}
```
##########
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java:
##########
@@ -743,6 +743,46 @@ public static String normalizeKey(String keyName,
return keyName;
}
+ /**
+ * Normalizes a given path up to the bucket level.
+ *
+ * This method takes a path as input and normalises uptil the bucket level.
+ * It handles empty, removes leading slashes, and splits the path into
+ * segments. It then extracts the volume and bucket names, forming a
+ * normalized path with a single slash. Finally, any remaining segments are
+ * joined as the key name, returning the complete standardized path.
+ *
+ * @param path The path string to be normalized.
+ * @return The normalized path string.
+ */
+ public static String normalizePathUptilBucket(String path) {
+ if (path == null || path.isEmpty()) {
+ return "/"; // Handle empty path
+ }
+
+ // Remove leading slashes
+ path = path.replaceAll("^/*", "");
+
+ String[] segments = path.split("/", -1);
+
+ String volumeName = segments[0];
+ String bucketName = segments.length > 1 ? segments[1] : "";
+
+ // Combine volume and bucket.
+ StringBuilder normalizedPath = new StringBuilder(volumeName);
+ if (!bucketName.isEmpty()) {
+ normalizedPath.append("/").append(bucketName);
+ }
+
+ // Add remaining segments as the key
+ if (segments.length > 2) {
+ normalizedPath.append("/").append(
+ String.join("/", Arrays.copyOfRange(segments, 2, segments.length)));
Review Comment:
```suggestion
String.join(OM_KEY_PREFIX, Arrays.copyOfRange(segments, 2,
segments.length)));
```
##########
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskWithLegacy.java:
##########
@@ -206,12 +131,118 @@ public boolean processWithLegacy(OMUpdateEventBatch
events) {
return true;
}
+ private void processWithFileSystemLayout(OmKeyInfo updatedKeyInfo,
+ OmKeyInfo oldKeyInfo,
+ OMDBUpdateEvent.OMDBUpdateAction
action,
+ Map<Long, NSSummary> nsSummaryMap)
+ throws IOException {
+ setKeyParentID(updatedKeyInfo);
+
+ if (!updatedKeyInfo.getKeyName().endsWith(OM_KEY_PREFIX)) {
+ switch (action) {
+ case PUT:
+ handlePutKeyEvent(updatedKeyInfo, nsSummaryMap);
+ break;
+
+ case DELETE:
+ handleDeleteKeyEvent(updatedKeyInfo, nsSummaryMap);
+ break;
+
+ case UPDATE:
+ if (oldKeyInfo != null) {
+ setKeyParentID(oldKeyInfo);
+ handleDeleteKeyEvent(oldKeyInfo, nsSummaryMap);
+ } else {
+ LOG.warn("Update event does not have the old keyInfo for {}.",
+ updatedKeyInfo.getKeyName());
+ }
+ handlePutKeyEvent(updatedKeyInfo, nsSummaryMap);
+ break;
+
+ default:
+ LOG.debug("Skipping DB update event fir Key: {}", action);
+ }
+ } else {
+ OmDirectoryInfo updatedDirectoryInfo = new OmDirectoryInfo.Builder()
+ .setName(updatedKeyInfo.getKeyName())
+ .setObjectID(updatedKeyInfo.getObjectID())
+ .setParentObjectID(updatedKeyInfo.getParentObjectID())
+ .build();
+
+ OmDirectoryInfo oldDirectoryInfo = null;
+
+ if (oldKeyInfo != null) {
+ oldDirectoryInfo =
+ new OmDirectoryInfo.Builder()
+ .setName(oldKeyInfo.getKeyName())
+ .setObjectID(oldKeyInfo.getObjectID())
+ .setParentObjectID(oldKeyInfo.getParentObjectID())
+ .build();
+ }
+
+ switch (action) {
+ case PUT:
+ handlePutDirEvent(updatedDirectoryInfo, nsSummaryMap);
+ break;
+
+ case DELETE:
+ handleDeleteDirEvent(updatedDirectoryInfo, nsSummaryMap);
+ break;
+
+ case UPDATE:
+ if (oldDirectoryInfo != null) {
+ handleDeleteDirEvent(oldDirectoryInfo, nsSummaryMap);
+ } else {
+ LOG.warn("Update event does not have the old dirInfo for {}.",
+ updatedKeyInfo.getKeyName());
+ }
+ handlePutDirEvent(updatedDirectoryInfo, nsSummaryMap);
+ break;
+
+ default:
+ LOG.debug("Skipping DB update event for Directory: {}", action);
Review Comment:
```suggestion
if(LOG.isDebugEnabled()) {
LOG.debug("Skipping DB update event for Directory: {}", action);
}
```
##########
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java:
##########
@@ -743,6 +743,46 @@ public static String normalizeKey(String keyName,
return keyName;
}
+ /**
+ * Normalizes a given path up to the bucket level.
+ *
+ * This method takes a path as input and normalises uptil the bucket level.
+ * It handles empty, removes leading slashes, and splits the path into
+ * segments. It then extracts the volume and bucket names, forming a
+ * normalized path with a single slash. Finally, any remaining segments are
+ * joined as the key name, returning the complete standardized path.
+ *
+ * @param path The path string to be normalized.
+ * @return The normalized path string.
+ */
+ public static String normalizePathUptilBucket(String path) {
+ if (path == null || path.isEmpty()) {
+ return "/"; // Handle empty path
+ }
+
+ // Remove leading slashes
+ path = path.replaceAll("^/*", "");
+
+ String[] segments = path.split("/", -1);
+
+ String volumeName = segments[0];
+ String bucketName = segments.length > 1 ? segments[1] : "";
+
+ // Combine volume and bucket.
+ StringBuilder normalizedPath = new StringBuilder(volumeName);
+ if (!bucketName.isEmpty()) {
+ normalizedPath.append("/").append(bucketName);
+ }
+
+ // Add remaining segments as the key
+ if (segments.length > 2) {
+ normalizedPath.append("/").append(
Review Comment:
```suggestion
normalizedPath.append(OM_KEY_PREFIX).append(
```
##########
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java:
##########
@@ -743,6 +743,46 @@ public static String normalizeKey(String keyName,
return keyName;
}
+ /**
+ * Normalizes a given path up to the bucket level.
+ *
+ * This method takes a path as input and normalises uptil the bucket level.
+ * It handles empty, removes leading slashes, and splits the path into
+ * segments. It then extracts the volume and bucket names, forming a
+ * normalized path with a single slash. Finally, any remaining segments are
+ * joined as the key name, returning the complete standardized path.
+ *
+ * @param path The path string to be normalized.
+ * @return The normalized path string.
+ */
+ public static String normalizePathUptilBucket(String path) {
+ if (path == null || path.isEmpty()) {
+ return "/"; // Handle empty path
+ }
+
+ // Remove leading slashes
+ path = path.replaceAll("^/*", "");
+
+ String[] segments = path.split("/", -1);
+
+ String volumeName = segments[0];
+ String bucketName = segments.length > 1 ? segments[1] : "";
+
+ // Combine volume and bucket.
+ StringBuilder normalizedPath = new StringBuilder(volumeName);
+ if (!bucketName.isEmpty()) {
+ normalizedPath.append("/").append(bucketName);
Review Comment:
```suggestion
normalizedPath.append(OM_KEY_PREFIX).append(bucketName);
```
##########
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskWithLegacy.java:
##########
@@ -300,17 +330,53 @@ private void setKeyParentID(OmKeyInfo keyInfo) throws
IOException {
"NSSummaryTaskWithLegacy is null");
}
} else {
- String bucketKey = getReconOMMetadataManager()
- .getBucketKey(keyInfo.getVolumeName(), keyInfo.getBucketName());
- OmBucketInfo parentBucketInfo =
- getReconOMMetadataManager().getBucketTable().getSkipCache(bucketKey);
+ setParentBucketId(keyInfo);
+ }
+ }
- if (parentBucketInfo != null) {
- keyInfo.setParentObjectID(parentBucketInfo.getObjectID());
- } else {
- throw new IOException("ParentKeyInfo for " +
- "NSSummaryTaskWithLegacy is null");
- }
+ /**
+ * Set the parent object ID for a bucket.
+ *@paramkeyInfo
+ *@throwsIOException
+ */
+ private void setParentBucketId(OmKeyInfo keyInfo)
+ throws IOException {
+ String bucketKey = getReconOMMetadataManager()
+ .getBucketKey(keyInfo.getVolumeName(), keyInfo.getBucketName());
+ OmBucketInfo parentBucketInfo =
+ getReconOMMetadataManager().getBucketTable().getSkipCache(bucketKey);
+
+ if (parentBucketInfo != null) {
+ keyInfo.setParentObjectID(parentBucketInfo.getObjectID());
+ } else {
+ throw new IOException("ParentKeyInfo for " +
+ "NSSummaryTaskWithLegacy is null");
}
}
+
+ /**
+ * Check if the bucket layout is LEGACY.
+ * @param metadataManager
+ * @param keyInfo
+ * @return
+ */
+ private boolean isBucketLayoutValid(ReconOMMetadataManager metadataManager,
+ OmKeyInfo keyInfo)
+ throws IOException {
+ String volumeName = keyInfo.getVolumeName();
+ String bucketName = keyInfo.getBucketName();
+ String bucketDBKey = metadataManager.getBucketKey(volumeName, bucketName);
+ OmBucketInfo omBucketInfo =
+ metadataManager.getBucketTable().getSkipCache(bucketDBKey);
+
+ if (omBucketInfo.getBucketLayout() != LEGACY_BUCKET_LAYOUT) {
+ LOG.debug(
Review Comment:
```suggestion
if(LOG.isDebugEnabled()) {
LOG.debug(
}
```
##########
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/EntityHandler.java:
##########
@@ -256,7 +280,52 @@ public static String[] parseRequestPath(String path) {
return names;
}
- private static String normalizePath(String path) {
+ /**
+ * Splits an object store path into volume, bucket, and key name components.
+ *
+ * This method parses a path of the format "/volumeName/bucketName/keyName",
+ * including paths with additional '/' characters within the key name. It's
+ * designed for object store paths where the first three '/' characters
+ * separate the root, volume and bucket names from the key name.
+ *
+ * @param path The object store path to parse, starting with a slash.
+ * @return A String array with three elements: volume name, bucket name, and
+ * key name, or {null} if the path format is invalid.
+ */
+ public static String[] parseObjectStorePath(String path) {
+ // Removing the leading slash for correct splitting
+ path = path.substring(1);
+
+ // Splitting the modified path by "/", limiting to 3 parts
+ String[] parts = path.split("/", 3);
+
+ // Checking if we correctly obtained 3 parts after removing the leading
slash
+ if (parts.length <= 3) {
+ return parts;
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Normalizes a given path based on the specified bucket layout.
+ *
+ * This method adjusts the path according to the bucket layout.
+ * For {OBJECT_STORE Layout}, it normalizes the path up to the bucket level
+ * using OmUtils.normalizePathUptilBucket. For other layouts, it
+ * normalizes the entire path, including the key, using
+ * OmUtils.normalizeKey, and does not preserve any trailing slashes.
+ * The normalized path will always be prefixed with OM_KEY_PREFIX to ensure
it
+ * is consistent with the expected format for object storage paths in Ozone.
+ *
+ * @param path
+ * @param bucketLayout
+ * @return A normalized path
+ */
+ private static String normalizePath(String path, BucketLayout bucketLayout) {
+ if (bucketLayout == BucketLayout.OBJECT_STORE) {
+ return OM_KEY_PREFIX + OmUtils.normalizePathUptilBucket(path);
Review Comment:
```suggestion
return OM_KEY_PREFIX + OmUtils.normalizePathUptoBucket(path);
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
