This is an automated email from the ASF dual-hosted git repository.
sumitagrawal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 7c38331ff1 HDDS-12110. Optimize memory overhead for OM background
tasks. (#7743)
7c38331ff1 is described below
commit 7c38331ff1e221fdf21763a1d8817a1e699ea5bf
Author: Devesh Kumar Singh <[email protected]>
AuthorDate: Thu Feb 13 09:10:31 2025 +0530
HDDS-12110. Optimize memory overhead for OM background tasks. (#7743)
---
.../spi/impl/OzoneManagerServiceProviderImpl.java | 5 +
.../recon/tasks/DeletedKeysInsightHandler.java | 40 +++-----
.../recon/tasks/NSSummaryTaskDbEventHandler.java | 21 ++--
.../hadoop/ozone/recon/tasks/OmTableHandler.java | 20 ++--
.../ozone/recon/tasks/OmTableInsightTask.java | 109 ++++++++++++---------
.../ozone/recon/tasks/OpenKeysInsightHandler.java | 46 ++++-----
.../hadoop/ozone/recon/tasks/ReconOmTask.java | 5 +
.../hadoop/ozone/recon/api/TestEndpoints.java | 2 +-
.../ozone/recon/tasks/TestOmTableInsightTask.java | 10 ++
9 files changed, 134 insertions(+), 124 deletions(-)
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java
index 72ce510019..1856657cd3 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java
@@ -69,6 +69,7 @@
import org.apache.hadoop.ozone.recon.spi.OzoneManagerServiceProvider;
import org.apache.hadoop.ozone.recon.tasks.OMDBUpdatesHandler;
import org.apache.hadoop.ozone.recon.tasks.OMUpdateEventBatch;
+import org.apache.hadoop.ozone.recon.tasks.ReconOmTask;
import org.apache.hadoop.ozone.recon.tasks.ReconTaskController;
import org.apache.hadoop.ozone.recon.tasks.updater.ReconTaskStatusUpdater;
import
org.apache.hadoop.ozone.recon.tasks.updater.ReconTaskStatusUpdaterManager;
@@ -266,6 +267,10 @@ public void start() {
ReconServerConfigKeys.RECON_OM_SNAPSHOT_TASK_INITIAL_DELAY,
OZONE_RECON_OM_SNAPSHOT_TASK_INITIAL_DELAY_DEFAULT),
TimeUnit.MILLISECONDS);
+ // Initialize recon om tasks for any first time initialization of
resources.
+ reconTaskController.getRegisteredTasks()
+ .values()
+ .forEach(ReconOmTask::init);
startSyncDataFromOM(initialDelay);
}
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/DeletedKeysInsightHandler.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/DeletedKeysInsightHandler.java
index 5a6d7a256e..14912cc920 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/DeletedKeysInsightHandler.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/DeletedKeysInsightHandler.java
@@ -27,7 +27,7 @@
import org.slf4j.LoggerFactory;
import java.io.IOException;
-import java.util.HashMap;
+import java.util.Map;
/**
* Manages records in the Deleted Table, updating counts and sizes of
@@ -45,23 +45,19 @@ public class DeletedKeysInsightHandler implements
OmTableHandler {
@Override
public void handlePutEvent(OMDBUpdateEvent<String, Object> event,
String tableName,
- HashMap<String, Long> objectCountMap,
- HashMap<String, Long> unReplicatedSizeMap,
- HashMap<String, Long> replicatedSizeMap) {
-
- String countKey = getTableCountKeyFromTable(tableName);
- String unReplicatedSizeKey = getUnReplicatedSizeKeyFromTable(tableName);
- String replicatedSizeKey = getReplicatedSizeKeyFromTable(tableName);
+ Map<String, Long> objectCountMap,
+ Map<String, Long> unReplicatedSizeMap,
+ Map<String, Long> replicatedSizeMap) {
if (event.getValue() != null) {
RepeatedOmKeyInfo repeatedOmKeyInfo =
(RepeatedOmKeyInfo) event.getValue();
- objectCountMap.computeIfPresent(countKey,
+ objectCountMap.computeIfPresent(getTableCountKeyFromTable(tableName),
(k, count) -> count + repeatedOmKeyInfo.getOmKeyInfoList().size());
Pair<Long, Long> result = repeatedOmKeyInfo.getTotalSize();
- unReplicatedSizeMap.computeIfPresent(unReplicatedSizeKey,
+
unReplicatedSizeMap.computeIfPresent(getUnReplicatedSizeKeyFromTable(tableName),
(k, size) -> size + result.getLeft());
- replicatedSizeMap.computeIfPresent(replicatedSizeKey,
+
replicatedSizeMap.computeIfPresent(getReplicatedSizeKeyFromTable(tableName),
(k, size) -> size + result.getRight());
} else {
LOG.warn("Put event does not have the Key Info for {}.",
@@ -77,23 +73,19 @@ public void handlePutEvent(OMDBUpdateEvent<String, Object>
event,
@Override
public void handleDeleteEvent(OMDBUpdateEvent<String, Object> event,
String tableName,
- HashMap<String, Long> objectCountMap,
- HashMap<String, Long> unReplicatedSizeMap,
- HashMap<String, Long> replicatedSizeMap) {
-
- String countKey = getTableCountKeyFromTable(tableName);
- String unReplicatedSizeKey = getUnReplicatedSizeKeyFromTable(tableName);
- String replicatedSizeKey = getReplicatedSizeKeyFromTable(tableName);
+ Map<String, Long> objectCountMap,
+ Map<String, Long> unReplicatedSizeMap,
+ Map<String, Long> replicatedSizeMap) {
if (event.getValue() != null) {
RepeatedOmKeyInfo repeatedOmKeyInfo =
(RepeatedOmKeyInfo) event.getValue();
- objectCountMap.computeIfPresent(countKey, (k, count) ->
+ objectCountMap.computeIfPresent(getTableCountKeyFromTable(tableName),
(k, count) ->
count > 0 ? count - repeatedOmKeyInfo.getOmKeyInfoList().size() :
0L);
Pair<Long, Long> result = repeatedOmKeyInfo.getTotalSize();
- unReplicatedSizeMap.computeIfPresent(unReplicatedSizeKey,
+
unReplicatedSizeMap.computeIfPresent(getUnReplicatedSizeKeyFromTable(tableName),
(k, size) -> size > result.getLeft() ? size - result.getLeft() : 0L);
- replicatedSizeMap.computeIfPresent(replicatedSizeKey,
+
replicatedSizeMap.computeIfPresent(getReplicatedSizeKeyFromTable(tableName),
(k, size) -> size > result.getRight() ? size - result.getRight() :
0L);
} else {
@@ -109,9 +101,9 @@ public void handleDeleteEvent(OMDBUpdateEvent<String,
Object> event,
@Override
public void handleUpdateEvent(OMDBUpdateEvent<String, Object> event,
String tableName,
- HashMap<String, Long> objectCountMap,
- HashMap<String, Long> unReplicatedSizeMap,
- HashMap<String, Long> replicatedSizeMap) {
+ Map<String, Long> objectCountMap,
+ Map<String, Long> unReplicatedSizeMap,
+ Map<String, Long> replicatedSizeMap) {
// The size of deleted keys cannot change hence no-op.
return;
}
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskDbEventHandler.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskDbEventHandler.java
index 888ec5319f..e0beefdc3d 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskDbEventHandler.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskDbEventHandler.java
@@ -100,13 +100,10 @@ protected void handlePutKeyEvent(OmKeyInfo keyInfo,
Map<Long,
// as this is a new ID
nsSummary = new NSSummary();
}
- int numOfFile = nsSummary.getNumOfFiles();
- long sizeOfFile = nsSummary.getSizeOfFiles();
int[] fileBucket = nsSummary.getFileSizeBucket();
- nsSummary.setNumOfFiles(numOfFile + 1);
- long dataSize = keyInfo.getDataSize();
- nsSummary.setSizeOfFiles(sizeOfFile + dataSize);
- int binIndex = ReconUtils.getFileSizeBinIndex(dataSize);
+ nsSummary.setNumOfFiles(nsSummary.getNumOfFiles() + 1);
+ nsSummary.setSizeOfFiles(nsSummary.getSizeOfFiles() +
keyInfo.getDataSize());
+ int binIndex = ReconUtils.getFileSizeBinIndex(keyInfo.getDataSize());
++fileBucket[binIndex];
nsSummary.setFileSizeBucket(fileBucket);
@@ -168,18 +165,15 @@ protected void handleDeleteKeyEvent(OmKeyInfo keyInfo,
LOG.error("The namespace table is not correctly populated.");
return;
}
- int numOfFile = nsSummary.getNumOfFiles();
- long sizeOfFile = nsSummary.getSizeOfFiles();
int[] fileBucket = nsSummary.getFileSizeBucket();
- long dataSize = keyInfo.getDataSize();
- int binIndex = ReconUtils.getFileSizeBinIndex(dataSize);
+ int binIndex = ReconUtils.getFileSizeBinIndex(keyInfo.getDataSize());
// decrement count, data size, and bucket count
// even if there's no direct key, we still keep the entry because
// we still need children dir IDs info
- nsSummary.setNumOfFiles(numOfFile - 1);
- nsSummary.setSizeOfFiles(sizeOfFile - dataSize);
+ nsSummary.setNumOfFiles(nsSummary.getNumOfFiles() - 1);
+ nsSummary.setSizeOfFiles(nsSummary.getSizeOfFiles() -
keyInfo.getDataSize());
--fileBucket[binIndex];
nsSummary.setFileSizeBucket(fileBucket);
nsSummaryMap.put(parentObjectId, nsSummary);
@@ -189,7 +183,6 @@ protected void handleDeleteDirEvent(OmDirectoryInfo
directoryInfo,
Map<Long, NSSummary> nsSummaryMap)
throws IOException {
long parentObjectId = directoryInfo.getParentObjectID();
- long objectId = directoryInfo.getObjectID();
// Try to get the NSSummary from our local map that maps NSSummaries to IDs
NSSummary nsSummary = nsSummaryMap.get(parentObjectId);
if (nsSummary == null) {
@@ -203,7 +196,7 @@ protected void handleDeleteDirEvent(OmDirectoryInfo
directoryInfo,
return;
}
- nsSummary.removeChildDir(objectId);
+ nsSummary.removeChildDir(directoryInfo.getObjectID());
nsSummaryMap.put(parentObjectId, nsSummary);
}
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OmTableHandler.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OmTableHandler.java
index 5ae23b68a7..b6a26248cb 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OmTableHandler.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OmTableHandler.java
@@ -23,7 +23,7 @@
import org.apache.hadoop.hdds.utils.db.TableIterator;
import java.io.IOException;
-import java.util.HashMap;
+import java.util.Map;
/**
* Interface for handling PUT, DELETE and UPDATE events for size-related
@@ -43,9 +43,9 @@ public interface OmTableHandler {
*/
void handlePutEvent(OMDBUpdateEvent<String, Object> event,
String tableName,
- HashMap<String, Long> objectCountMap,
- HashMap<String, Long> unReplicatedSizeMap,
- HashMap<String, Long> replicatedSizeMap);
+ Map<String, Long> objectCountMap,
+ Map<String, Long> unReplicatedSizeMap,
+ Map<String, Long> replicatedSizeMap);
/**
@@ -60,9 +60,9 @@ void handlePutEvent(OMDBUpdateEvent<String, Object> event,
*/
void handleDeleteEvent(OMDBUpdateEvent<String, Object> event,
String tableName,
- HashMap<String, Long> objectCountMap,
- HashMap<String, Long> unReplicatedSizeMap,
- HashMap<String, Long> replicatedSizeMap);
+ Map<String, Long> objectCountMap,
+ Map<String, Long> unReplicatedSizeMap,
+ Map<String, Long> replicatedSizeMap);
/**
@@ -77,9 +77,9 @@ void handleDeleteEvent(OMDBUpdateEvent<String, Object> event,
*/
void handleUpdateEvent(OMDBUpdateEvent<String, Object> event,
String tableName,
- HashMap<String, Long> objectCountMap,
- HashMap<String, Long> unReplicatedSizeMap,
- HashMap<String, Long> replicatedSizeMap);
+ Map<String, Long> objectCountMap,
+ Map<String, Long> unReplicatedSizeMap,
+ Map<String, Long> replicatedSizeMap);
/**
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OmTableInsightTask.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OmTableInsightTask.java
index 37a0e16e93..30d4a1c4e1 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OmTableInsightTask.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OmTableInsightTask.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.recon.tasks;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Iterators;
import com.google.inject.Inject;
import org.apache.commons.lang3.tuple.ImmutablePair;
@@ -62,6 +63,11 @@ public class OmTableInsightTask implements ReconOmTask {
private Configuration sqlConfiguration;
private ReconOMMetadataManager reconOMMetadataManager;
private Map<String, OmTableHandler> tableHandlers;
+ private Collection<String> tables;
+ private Map<String, Long> objectCountMap;
+ private Map<String, Long> unReplicatedSizeMap;
+ private Map<String, Long> replicatedSizeMap;
+
@Inject
public OmTableInsightTask(GlobalStatsDao globalStatsDao,
@@ -78,6 +84,20 @@ public OmTableInsightTask(GlobalStatsDao globalStatsDao,
tableHandlers.put(DELETED_TABLE, new DeletedKeysInsightHandler());
}
+ /**
+ * Initialize the OM table insight task with first time initialization of
resources.
+ */
+ @Override
+ public void init() {
+ ReconOmTask.super.init();
+ tables = getTaskTables();
+
+ // Initialize maps to store count and size information
+ objectCountMap = initializeCountMap();
+ unReplicatedSizeMap = initializeSizeMap(false);
+ replicatedSizeMap = initializeSizeMap(true);
+ }
+
/**
* Iterates the rows of each table in the OM snapshot DB and calculates the
* counts and sizes for table data.
@@ -92,16 +112,9 @@ public OmTableInsightTask(GlobalStatsDao globalStatsDao,
*/
@Override
public Pair<String, Boolean> reprocess(OMMetadataManager omMetadataManager) {
- HashMap<String, Long> objectCountMap = initializeCountMap();
- HashMap<String, Long> unReplicatedSizeMap = initializeSizeMap(false);
- HashMap<String, Long> replicatedSizeMap = initializeSizeMap(true);
-
- for (String tableName : getTaskTables()) {
+ init();
+ for (String tableName : tables) {
Table table = omMetadataManager.getTable(tableName);
- if (table == null) {
- LOG.error("Table " + tableName + " not found in OM Metadata.");
- return new ImmutablePair<>(getTaskName(), false);
- }
try (TableIterator<String, ? extends Table.KeyValue<String, ?>> iterator
= table.iterator()) {
@@ -157,35 +170,29 @@ public Collection<String> getTaskTables() {
@Override
public Pair<String, Boolean> process(OMUpdateEventBatch events) {
Iterator<OMDBUpdateEvent> eventIterator = events.getIterator();
- // Initialize maps to store count and size information
- HashMap<String, Long> objectCountMap = initializeCountMap();
- HashMap<String, Long> unReplicatedSizeMap = initializeSizeMap(false);
- HashMap<String, Long> replicatedSizeMap = initializeSizeMap(true);
- final Collection<String> taskTables = getTaskTables();
+ String tableName;
+ OMDBUpdateEvent<String, Object> omdbUpdateEvent;
// Process each update event
long startTime = System.currentTimeMillis();
while (eventIterator.hasNext()) {
- OMDBUpdateEvent<String, Object> omdbUpdateEvent = eventIterator.next();
- String tableName = omdbUpdateEvent.getTable();
- if (!taskTables.contains(tableName)) {
+ omdbUpdateEvent = eventIterator.next();
+ tableName = omdbUpdateEvent.getTable();
+ if (!tables.contains(tableName)) {
continue;
}
try {
switch (omdbUpdateEvent.getAction()) {
case PUT:
- handlePutEvent(omdbUpdateEvent, tableName, objectCountMap,
- unReplicatedSizeMap, replicatedSizeMap);
+ handlePutEvent(omdbUpdateEvent, tableName);
break;
case DELETE:
- handleDeleteEvent(omdbUpdateEvent, tableName, objectCountMap,
- unReplicatedSizeMap, replicatedSizeMap);
+ handleDeleteEvent(omdbUpdateEvent, tableName);
break;
case UPDATE:
- handleUpdateEvent(omdbUpdateEvent, tableName, objectCountMap,
- unReplicatedSizeMap, replicatedSizeMap);
+ handleUpdateEvent(omdbUpdateEvent, tableName);
break;
default:
@@ -215,11 +222,7 @@ public Pair<String, Boolean> process(OMUpdateEventBatch
events) {
}
private void handlePutEvent(OMDBUpdateEvent<String, Object> event,
- String tableName,
- HashMap<String, Long> objectCountMap,
- HashMap<String, Long> unReplicatedSizeMap,
- HashMap<String, Long> replicatedSizeMap)
- throws IOException {
+ String tableName) {
OmTableHandler tableHandler = tableHandlers.get(tableName);
if (event.getValue() != null) {
if (tableHandler != null) {
@@ -234,19 +237,14 @@ private void handlePutEvent(OMDBUpdateEvent<String,
Object> event,
private void handleDeleteEvent(OMDBUpdateEvent<String, Object> event,
- String tableName,
- HashMap<String, Long> objectCountMap,
- HashMap<String, Long> unReplicatedSizeMap,
- HashMap<String, Long> replicatedSizeMap)
- throws IOException {
+ String tableName) {
OmTableHandler tableHandler = tableHandlers.get(tableName);
if (event.getValue() != null) {
if (tableHandler != null) {
tableHandler.handleDeleteEvent(event, tableName, objectCountMap,
unReplicatedSizeMap, replicatedSizeMap);
} else {
- String countKey = getTableCountKeyFromTable(tableName);
- objectCountMap.computeIfPresent(countKey,
+ objectCountMap.computeIfPresent(getTableCountKeyFromTable(tableName),
(k, count) -> count > 0 ? count - 1L : 0L);
}
}
@@ -254,10 +252,7 @@ private void handleDeleteEvent(OMDBUpdateEvent<String,
Object> event,
private void handleUpdateEvent(OMDBUpdateEvent<String, Object> event,
- String tableName,
- HashMap<String, Long> objectCountMap,
- HashMap<String, Long> unReplicatedSizeMap,
- HashMap<String, Long> replicatedSizeMap) {
+ String tableName) {
OmTableHandler tableHandler = tableHandlers.get(tableName);
if (event.getValue() != null) {
@@ -302,14 +297,13 @@ private void writeDataToDB(Map<String, Long> dataMap) {
*
* @return The count map containing the counts for each table.
*/
- private HashMap<String, Long> initializeCountMap() {
- Collection<String> tables = getTaskTables();
- HashMap<String, Long> objectCountMap = new HashMap<>(tables.size());
+ public HashMap<String, Long> initializeCountMap() {
+ HashMap<String, Long> objCountMap = new HashMap<>(tables.size());
for (String tableName : tables) {
String key = getTableCountKeyFromTable(tableName);
- objectCountMap.put(key, getValueForKey(key));
+ objCountMap.put(key, getValueForKey(key));
}
- return objectCountMap;
+ return objCountMap;
}
/**
@@ -318,11 +312,13 @@ private HashMap<String, Long> initializeCountMap() {
*
* @return The size map containing the size counts for each table.
*/
- private HashMap<String, Long> initializeSizeMap(boolean replicated) {
+ public HashMap<String, Long> initializeSizeMap(boolean replicated) {
+ String tableName;
+ OmTableHandler tableHandler;
HashMap<String, Long> sizeCountMap = new HashMap<>();
for (Map.Entry<String, OmTableHandler> entry : tableHandlers.entrySet()) {
- String tableName = entry.getKey();
- OmTableHandler tableHandler = entry.getValue();
+ tableName = entry.getKey();
+ tableHandler = entry.getValue();
String key =
replicated ? tableHandler.getReplicatedSizeKeyFromTable(tableName) :
tableHandler.getUnReplicatedSizeKeyFromTable(tableName);
@@ -356,6 +352,25 @@ private long getValueForKey(String key) {
return (record == null) ? 0L : record.getValue();
}
+ @VisibleForTesting
+ public void setTables(Collection<String> tables) {
+ this.tables = tables;
+ }
+
+ @VisibleForTesting
+ public void setObjectCountMap(HashMap<String, Long> objectCountMap) {
+ this.objectCountMap = objectCountMap;
+ }
+
+ @VisibleForTesting
+ public void setUnReplicatedSizeMap(HashMap<String, Long>
unReplicatedSizeMap) {
+ this.unReplicatedSizeMap = unReplicatedSizeMap;
+ }
+
+ @VisibleForTesting
+ public void setReplicatedSizeMap(HashMap<String, Long> replicatedSizeMap) {
+ this.replicatedSizeMap = replicatedSizeMap;
+ }
}
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OpenKeysInsightHandler.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OpenKeysInsightHandler.java
index 7a27d29d8f..2316ae678d 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OpenKeysInsightHandler.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OpenKeysInsightHandler.java
@@ -26,7 +26,7 @@
import org.slf4j.LoggerFactory;
import java.io.IOException;
-import java.util.HashMap;
+import java.util.Map;
/**
* Manages records in the OpenKey Table, updating counts and sizes of
@@ -44,20 +44,16 @@ public class OpenKeysInsightHandler implements
OmTableHandler {
@Override
public void handlePutEvent(OMDBUpdateEvent<String, Object> event,
String tableName,
- HashMap<String, Long> objectCountMap,
- HashMap<String, Long> unReplicatedSizeMap,
- HashMap<String, Long> replicatedSizeMap) {
-
- String countKey = getTableCountKeyFromTable(tableName);
- String unReplicatedSizeKey = getUnReplicatedSizeKeyFromTable(tableName);
- String replicatedSizeKey = getReplicatedSizeKeyFromTable(tableName);
+ Map<String, Long> objectCountMap,
+ Map<String, Long> unReplicatedSizeMap,
+ Map<String, Long> replicatedSizeMap) {
if (event.getValue() != null) {
OmKeyInfo omKeyInfo = (OmKeyInfo) event.getValue();
- objectCountMap.computeIfPresent(countKey, (k, count) -> count + 1L);
- unReplicatedSizeMap.computeIfPresent(unReplicatedSizeKey,
+ objectCountMap.computeIfPresent(getTableCountKeyFromTable(tableName),
(k, count) -> count + 1L);
+
unReplicatedSizeMap.computeIfPresent(getUnReplicatedSizeKeyFromTable(tableName),
(k, size) -> size + omKeyInfo.getDataSize());
- replicatedSizeMap.computeIfPresent(replicatedSizeKey,
+
replicatedSizeMap.computeIfPresent(getReplicatedSizeKeyFromTable(tableName),
(k, size) -> size + omKeyInfo.getReplicatedSize());
} else {
LOG.warn("Put event does not have the Key Info for {}.",
@@ -72,22 +68,18 @@ public void handlePutEvent(OMDBUpdateEvent<String, Object>
event,
@Override
public void handleDeleteEvent(OMDBUpdateEvent<String, Object> event,
String tableName,
- HashMap<String, Long> objectCountMap,
- HashMap<String, Long> unReplicatedSizeMap,
- HashMap<String, Long> replicatedSizeMap) {
-
- String countKey = getTableCountKeyFromTable(tableName);
- String unReplicatedSizeKey = getUnReplicatedSizeKeyFromTable(tableName);
- String replicatedSizeKey = getReplicatedSizeKeyFromTable(tableName);
+ Map<String, Long> objectCountMap,
+ Map<String, Long> unReplicatedSizeMap,
+ Map<String, Long> replicatedSizeMap) {
if (event.getValue() != null) {
OmKeyInfo omKeyInfo = (OmKeyInfo) event.getValue();
- objectCountMap.computeIfPresent(countKey,
+ objectCountMap.computeIfPresent(getTableCountKeyFromTable(tableName),
(k, count) -> count > 0 ? count - 1L : 0L);
- unReplicatedSizeMap.computeIfPresent(unReplicatedSizeKey,
+
unReplicatedSizeMap.computeIfPresent(getUnReplicatedSizeKeyFromTable(tableName),
(k, size) -> size > omKeyInfo.getDataSize() ?
size - omKeyInfo.getDataSize() : 0L);
- replicatedSizeMap.computeIfPresent(replicatedSizeKey,
+
replicatedSizeMap.computeIfPresent(getReplicatedSizeKeyFromTable(tableName),
(k, size) -> size > omKeyInfo.getReplicatedSize() ?
size - omKeyInfo.getReplicatedSize() : 0L);
} else {
@@ -103,9 +95,9 @@ public void handleDeleteEvent(OMDBUpdateEvent<String,
Object> event,
@Override
public void handleUpdateEvent(OMDBUpdateEvent<String, Object> event,
String tableName,
- HashMap<String, Long> objectCountMap,
- HashMap<String, Long> unReplicatedSizeMap,
- HashMap<String, Long> replicatedSizeMap) {
+ Map<String, Long> objectCountMap,
+ Map<String, Long> unReplicatedSizeMap,
+ Map<String, Long> replicatedSizeMap) {
if (event.getValue() != null) {
if (event.getOldValue() == null) {
@@ -113,17 +105,15 @@ public void handleUpdateEvent(OMDBUpdateEvent<String,
Object> event,
event.getKey());
return;
}
- String unReplicatedSizeKey = getUnReplicatedSizeKeyFromTable(tableName);
- String replicatedSizeKey = getReplicatedSizeKeyFromTable(tableName);
// In Update event the count for the open table will not change. So we
// don't need to update the count.
OmKeyInfo oldKeyInfo = (OmKeyInfo) event.getOldValue();
OmKeyInfo newKeyInfo = (OmKeyInfo) event.getValue();
- unReplicatedSizeMap.computeIfPresent(unReplicatedSizeKey,
+
unReplicatedSizeMap.computeIfPresent(getUnReplicatedSizeKeyFromTable(tableName),
(k, size) -> size - oldKeyInfo.getDataSize() +
newKeyInfo.getDataSize());
- replicatedSizeMap.computeIfPresent(replicatedSizeKey,
+
replicatedSizeMap.computeIfPresent(getReplicatedSizeKeyFromTable(tableName),
(k, size) -> size - oldKeyInfo.getReplicatedSize() +
newKeyInfo.getReplicatedSize());
} else {
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ReconOmTask.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ReconOmTask.java
index 2092d6a326..75ebebb334 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ReconOmTask.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ReconOmTask.java
@@ -32,6 +32,11 @@ public interface ReconOmTask {
*/
String getTaskName();
+ /**
+ * Initialize the recon om task with first time initialization of resources.
+ */
+ default void init() { }
+
/**
* Process a set of OM events on tables that the task is listening on.
* @param events Set of events to be processed by the task.
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java
index 5b6d22bf26..7e151640c7 100644
---
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java
@@ -786,7 +786,7 @@ public void testGetClusterState() throws Exception {
(ClusterStateResponse) response1.getEntity();
return (clusterStateResponse1.getContainers() == 1);
});
-
+ omTableInsightTask.init();
// check volume, bucket and key count after running table count task
Pair<String, Boolean> result =
omTableInsightTask.reprocess(reconOMMetadataManager);
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOmTableInsightTask.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOmTableInsightTask.java
index 8bb4c1f724..b83b846ce5 100644
---
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOmTableInsightTask.java
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOmTableInsightTask.java
@@ -44,6 +44,8 @@
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.junit.jupiter.api.io.TempDir;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.file.Files;
@@ -127,6 +129,9 @@ public class TestOmTableInsightTask extends
AbstractReconSqlDBTest {
@Mock
private Table<Long, NSSummary> nsSummaryTable;
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestOmTableInsightTask.class);
+
public TestOmTableInsightTask() {
super();
}
@@ -154,6 +159,11 @@ private void initializeInjector() throws IOException {
reconNamespaceSummaryManager, reconOMMetadataManager,
ozoneConfiguration);
dslContext = getDslContext();
+
+ omTableInsightTask.setTables(omTableInsightTask.getTaskTables());
+
omTableInsightTask.setObjectCountMap(omTableInsightTask.initializeCountMap());
+
omTableInsightTask.setUnReplicatedSizeMap(omTableInsightTask.initializeSizeMap(false));
+
omTableInsightTask.setReplicatedSizeMap(omTableInsightTask.initializeSizeMap(true));
}
@BeforeEach
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
