devmadhuu commented on code in PR #5037:
URL: https://github.com/apache/ozone/pull/5037#discussion_r1406118882
##########
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/DeletedTableHandler.java:
##########
@@ -0,0 +1,176 @@
+package org.apache.hadoop.ozone.recon.tasks;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.commons.lang3.tuple.Triple;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+
+/**
+ * Manages records in the Deleted Table, updating counts and sizes of
+ * pending Key Deletions in the backend.
+ */
+public class DeletedTableHandler implements OmTableHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DeletedTableHandler.class);
+
+ /**
+ * Invoked by the process method to add information on those keys that have
+ * been backlogged in the backend for deletion.
+ */
+ @Override
+ public void handlePutEvent(OMDBUpdateEvent<String, Object> event,
+ String tableName,
+ Collection<String> sizeRelatedTables,
+ HashMap<String, Long> objectCountMap,
+ HashMap<String, Long> unreplicatedSizeCountMap,
+ HashMap<String, Long> replicatedSizeCountMap)
+ throws IOException {
+
+ String countKey = getTableCountKeyFromTable(tableName);
+ String unReplicatedSizeKey = getUnReplicatedSizeKeyFromTable(tableName);
+ String replicatedSizeKey = getReplicatedSizeKeyFromTable(tableName);
+
+ if (event.getValue() != null) {
+ RepeatedOmKeyInfo repeatedOmKeyInfo =
+ (RepeatedOmKeyInfo) event.getValue();
+ objectCountMap.computeIfPresent(countKey,
+ (k, count) -> count + repeatedOmKeyInfo.getOmKeyInfoList().size());
+ Pair<Long, Long> result = repeatedOmKeyInfo.getTotalSize();
+ unreplicatedSizeCountMap.computeIfPresent(unReplicatedSizeKey,
+ (k, size) -> size + result.getLeft());
+ replicatedSizeCountMap.computeIfPresent(replicatedSizeKey,
+ (k, size) -> size + result.getRight());
+ } else {
+ LOG.warn("Put event does not have the Key Info for {}.",
+ event.getKey());
+ }
+
+ }
+
+ /**
+ * Invoked by the process method to remove information on those keys that
have
+ * been successfully deleted from the backend.
+ */
+ @Override
+ public void handleDeleteEvent(OMDBUpdateEvent<String, Object> event,
+ String tableName,
+ Collection<String> sizeRelatedTables,
+ HashMap<String, Long> objectCountMap,
+ HashMap<String, Long> unreplicatedSizeCountMap,
+ HashMap<String, Long> replicatedSizeCountMap)
+ throws IOException {
+
+ String countKey = getTableCountKeyFromTable(tableName);
+ String unReplicatedSizeKey = getUnReplicatedSizeKeyFromTable(tableName);
+ String replicatedSizeKey = getReplicatedSizeKeyFromTable(tableName);
+
+ if (event.getValue() != null) {
+ RepeatedOmKeyInfo repeatedOmKeyInfo =
+ (RepeatedOmKeyInfo) event.getValue();
+ objectCountMap.computeIfPresent(countKey, (k, count) ->
+ count > 0 ? count - repeatedOmKeyInfo.getOmKeyInfoList().size() :
0L);
+ Pair<Long, Long> result = repeatedOmKeyInfo.getTotalSize();
+ unreplicatedSizeCountMap.computeIfPresent(unReplicatedSizeKey,
+ (k, size) -> size > result.getLeft() ? size - result.getLeft() : 0L);
+ replicatedSizeCountMap.computeIfPresent(replicatedSizeKey,
+ (k, size) -> size > result.getRight() ? size - result.getRight() :
+ 0L);
+ } else {
+ LOG.warn("Delete event does not have the Key Info for {}.",
+ event.getKey());
+ }
+ }
+
+ /**
+ * Invoked by the process method to update the statistics on the keys
+ * pending to be deleted.
+ */
+ @Override
+ public void handleUpdateEvent(OMDBUpdateEvent<String, Object> event,
Review Comment:
Can you pls specify the business use case when any update event will be
there for deleted table ? Do OM perform any update operations on deletedTable ?
##########
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/DeletedTableHandler.java:
##########
@@ -0,0 +1,176 @@
+package org.apache.hadoop.ozone.recon.tasks;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.commons.lang3.tuple.Triple;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+
+/**
+ * Manages records in the Deleted Table, updating counts and sizes of
+ * pending Key Deletions in the backend.
+ */
+public class DeletedTableHandler implements OmTableHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DeletedTableHandler.class);
+
+ /**
+ * Invoked by the process method to add information on those keys that have
+ * been backlogged in the backend for deletion.
+ */
+ @Override
+ public void handlePutEvent(OMDBUpdateEvent<String, Object> event,
+ String tableName,
+ Collection<String> sizeRelatedTables,
+ HashMap<String, Long> objectCountMap,
+ HashMap<String, Long> unreplicatedSizeCountMap,
+ HashMap<String, Long> replicatedSizeCountMap)
+ throws IOException {
+
+ String countKey = getTableCountKeyFromTable(tableName);
+ String unReplicatedSizeKey = getUnReplicatedSizeKeyFromTable(tableName);
+ String replicatedSizeKey = getReplicatedSizeKeyFromTable(tableName);
+
+ if (event.getValue() != null) {
+ RepeatedOmKeyInfo repeatedOmKeyInfo =
+ (RepeatedOmKeyInfo) event.getValue();
+ objectCountMap.computeIfPresent(countKey,
+ (k, count) -> count + repeatedOmKeyInfo.getOmKeyInfoList().size());
+ Pair<Long, Long> result = repeatedOmKeyInfo.getTotalSize();
+ unreplicatedSizeCountMap.computeIfPresent(unReplicatedSizeKey,
+ (k, size) -> size + result.getLeft());
+ replicatedSizeCountMap.computeIfPresent(replicatedSizeKey,
+ (k, size) -> size + result.getRight());
+ } else {
+ LOG.warn("Put event does not have the Key Info for {}.",
+ event.getKey());
+ }
+
+ }
+
+ /**
+ * Invoked by the process method to remove information on those keys that
have
+ * been successfully deleted from the backend.
+ */
+ @Override
+ public void handleDeleteEvent(OMDBUpdateEvent<String, Object> event,
Review Comment:
Because this service is not snapshot aware, handling delete may not give
correct info how many files pending for delete and the amount of data to be
reclaimed. Because if a key is deleted and present in deletedTable and at this
point if snapshot taken then the key will be removed from active DB table and
still hold and occupying space in snapshot DB table.
##########
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/DeletedDirectoryTableHandler.java:
##########
@@ -0,0 +1,165 @@
+package org.apache.hadoop.ozone.recon.tasks;
+
+import org.apache.commons.lang3.tuple.Triple;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.recon.api.types.NSSummary;
+import org.apache.hadoop.ozone.recon.spi.impl.ReconNamespaceSummaryManagerImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Manages records in the Deleted Directory Table, updating counts and sizes of
+ * pending Directory Deletions in the backend.
+ */
+public class DeletedDirectoryTableHandler implements OmTableHandler {
+
+ private ReconNamespaceSummaryManagerImpl reconNamespaceSummaryManager;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DeletedTableHandler.class);
+
+ public DeletedDirectoryTableHandler(
+ ReconNamespaceSummaryManagerImpl reconNamespaceSummaryManager) {
+ this.reconNamespaceSummaryManager = reconNamespaceSummaryManager;
+ }
+
+ /**
+ * Invoked by the process method to add information on those directories that
+ * have been backlogged in the backend for deletion.
+ */
+ @Override
+ public void handlePutEvent(OMDBUpdateEvent<String, Object> event,
+ String tableName,
+ Collection<String> sizeRelatedTables,
+ HashMap<String, Long> objectCountMap,
+ HashMap<String, Long> unreplicatedSizeCountMap,
+ HashMap<String, Long> replicatedSizeCountMap)
+ throws IOException {
+ String countKey = getTableCountKeyFromTable(tableName);
+ String unReplicatedSizeKey = getUnReplicatedSizeKeyFromTable(tableName);
+
+ if (event.getValue() != null) {
+ OmKeyInfo omKeyInfo = (OmKeyInfo) event.getValue();
+ objectCountMap.computeIfPresent(countKey, (k, count) -> count + 1L);
+ Long newDeletedDirectorySize =
+ fetchSizeForDeletedDirectory(omKeyInfo.getObjectID());
+ unreplicatedSizeCountMap.computeIfPresent(unReplicatedSizeKey,
Review Comment:
Based on replication config, pls check replicatedSize can be computed.
##########
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/DeletedDirectoryTableHandler.java:
##########
@@ -0,0 +1,165 @@
+package org.apache.hadoop.ozone.recon.tasks;
+
+import org.apache.commons.lang3.tuple.Triple;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.recon.api.types.NSSummary;
+import org.apache.hadoop.ozone.recon.spi.impl.ReconNamespaceSummaryManagerImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Manages records in the Deleted Directory Table, updating counts and sizes of
+ * pending Directory Deletions in the backend.
+ */
+public class DeletedDirectoryTableHandler implements OmTableHandler {
+
+ private ReconNamespaceSummaryManagerImpl reconNamespaceSummaryManager;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DeletedTableHandler.class);
+
+ public DeletedDirectoryTableHandler(
+ ReconNamespaceSummaryManagerImpl reconNamespaceSummaryManager) {
+ this.reconNamespaceSummaryManager = reconNamespaceSummaryManager;
+ }
+
+ /**
+ * Invoked by the process method to add information on those directories that
+ * have been backlogged in the backend for deletion.
+ */
+ @Override
+ public void handlePutEvent(OMDBUpdateEvent<String, Object> event,
+ String tableName,
+ Collection<String> sizeRelatedTables,
+ HashMap<String, Long> objectCountMap,
+ HashMap<String, Long> unreplicatedSizeCountMap,
+ HashMap<String, Long> replicatedSizeCountMap)
+ throws IOException {
+ String countKey = getTableCountKeyFromTable(tableName);
+ String unReplicatedSizeKey = getUnReplicatedSizeKeyFromTable(tableName);
+
+ if (event.getValue() != null) {
+ OmKeyInfo omKeyInfo = (OmKeyInfo) event.getValue();
+ objectCountMap.computeIfPresent(countKey, (k, count) -> count + 1L);
+ Long newDeletedDirectorySize =
+ fetchSizeForDeletedDirectory(omKeyInfo.getObjectID());
+ unreplicatedSizeCountMap.computeIfPresent(unReplicatedSizeKey,
+ (k, size) -> size + newDeletedDirectorySize);
+ } else {
+ LOG.warn("Put event does not have the Key Info for {}.",
+ event.getKey());
+ }
+ }
+
+ /**
+ * Invoked by the process method to remove information on those directories
+ * that have been successfully deleted from the backend.
+ */
+ @Override
+ public void handleDeleteEvent(OMDBUpdateEvent<String, Object> event,
+ String tableName,
+ Collection<String> sizeRelatedTables,
+ HashMap<String, Long> objectCountMap,
+ HashMap<String, Long> unreplicatedSizeCountMap,
+ HashMap<String, Long> replicatedSizeCountMap)
+ throws IOException {
+ String countKey = getTableCountKeyFromTable(tableName);
+ String unReplicatedSizeKey = getUnReplicatedSizeKeyFromTable(tableName);
+
+ if (event.getValue() != null) {
+ OmKeyInfo omKeyInfo = (OmKeyInfo) event.getValue();
+ objectCountMap.computeIfPresent(countKey, (k, count) -> count - 1L);
+ Long newDeletedDirectorySize =
+ fetchSizeForDeletedDirectory(omKeyInfo.getObjectID());
Review Comment:
Same comment related to Snapshot aware as above.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
