This is an automated email from the ASF dual-hosted git repository.
zitadombi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 2d9d92197f HDDS-9052. The logs in Recon are flooded with excessive
warning messages, resulting in log overload (#5100)
2d9d92197f is described below
commit 2d9d92197fd1a07163e274b594e69f2a846c7f39
Author: Arafat2198 <[email protected]>
AuthorDate: Mon Jul 24 19:11:27 2023 +0530
HDDS-9052. The logs in Recon are flooded with excessive warning messages,
resulting in log overload (#5100)
---
.../ozone/recon/tasks/OmTableInsightTask.java | 74 ++++++++++++++--------
1 file changed, 49 insertions(+), 25 deletions(-)
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OmTableInsightTask.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OmTableInsightTask.java
index c28f5949f1..9a009d9ee0 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OmTableInsightTask.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OmTableInsightTask.java
@@ -272,27 +272,29 @@ public class OmTableInsightTask implements ReconOmTask {
HashMap<String, Long> objectCountMap,
HashMap<String, Long> unreplicatedSizeCountMap,
HashMap<String, Long> replicatedSizeCountMap) {
- String countKey = getTableCountKeyFromTable(tableName);
- String unReplicatedSizeKey = getUnReplicatedSizeKeyFromTable(tableName);
- String replicatedSizeKey = getReplicatedSizeKeyFromTable(tableName);
if (sizeRelatedTables.contains(tableName)) {
- handleSizeRelatedTablePutEvent(event, countKey, unReplicatedSizeKey,
- replicatedSizeKey, objectCountMap,
+ handleSizeRelatedTablePutEvent(event, tableName, objectCountMap,
unreplicatedSizeCountMap, replicatedSizeCountMap);
} else {
+ String countKey = getTableCountKeyFromTable(tableName);
objectCountMap.computeIfPresent(countKey, (k, count) -> count + 1L);
}
}
private void handleSizeRelatedTablePutEvent(
- OMDBUpdateEvent<String, Object> event, String countKey,
- String unReplicatedSizeKey, String replicatedSizeKey,
+ OMDBUpdateEvent<String, Object> event,
+ String tableName,
HashMap<String, Long> objectCountMap,
HashMap<String, Long> unreplicatedSizeCountMap,
HashMap<String, Long> replicatedSizeCountMap) {
- // Handle PUT for OpenKeyTable & OpenFileTable
+
+ String countKey = getTableCountKeyFromTable(tableName);
+ String unReplicatedSizeKey = getUnReplicatedSizeKeyFromTable(tableName);
+ String replicatedSizeKey = getReplicatedSizeKeyFromTable(tableName);
+
if (event.getValue() instanceof OmKeyInfo) {
+ // Handle PUT for OpenKeyTable & OpenFileTable
OmKeyInfo omKeyInfo = (OmKeyInfo) event.getValue();
objectCountMap.computeIfPresent(countKey, (k, count) -> count + 1L);
unreplicatedSizeCountMap.computeIfPresent(unReplicatedSizeKey,
@@ -313,22 +315,20 @@ public class OmTableInsightTask implements ReconOmTask {
}
}
+
private void handleDeleteEvent(OMDBUpdateEvent<String, Object> event,
String tableName,
Collection<String> sizeRelatedTables,
HashMap<String, Long> objectCountMap,
HashMap<String, Long>
unreplicatedSizeCountMap,
HashMap<String, Long> replicatedSizeCountMap)
{
- String countKey = getTableCountKeyFromTable(tableName);
- String unReplicatedSizeKey = getUnReplicatedSizeKeyFromTable(tableName);
- String replicatedSizeKey = getReplicatedSizeKeyFromTable(tableName);
if (event.getValue() != null) {
if (sizeRelatedTables.contains(tableName)) {
- handleSizeRelatedTableDeleteEvent(event, countKey, unReplicatedSizeKey,
- replicatedSizeKey, objectCountMap,
+ handleSizeRelatedTableDeleteEvent(event, tableName, objectCountMap,
unreplicatedSizeCountMap, replicatedSizeCountMap);
} else {
+ String countKey = getTableCountKeyFromTable(tableName);
objectCountMap.computeIfPresent(countKey,
(k, count) -> count > 0 ? count - 1L : 0L);
}
@@ -336,11 +336,16 @@ public class OmTableInsightTask implements ReconOmTask {
}
private void handleSizeRelatedTableDeleteEvent(
- OMDBUpdateEvent<String, Object> event, String countKey,
- String unReplicatedSizeKey, String replicatedSizeKey,
+ OMDBUpdateEvent<String, Object> event,
+ String tableName,
HashMap<String, Long> objectCountMap,
HashMap<String, Long> unreplicatedSizeCountMap,
HashMap<String, Long> replicatedSizeCountMap) {
+
+ String countKey = getTableCountKeyFromTable(tableName);
+ String unReplicatedSizeKey = getUnReplicatedSizeKeyFromTable(tableName);
+ String replicatedSizeKey = getReplicatedSizeKeyFromTable(tableName);
+
if (event.getValue() instanceof OmKeyInfo) {
// Handle DELETE for OpenKeyTable & OpenFileTable
OmKeyInfo omKeyInfo = (OmKeyInfo) event.getValue();
@@ -356,8 +361,8 @@ public class OmTableInsightTask implements ReconOmTask {
// Handle DELETE for DeletedTable
RepeatedOmKeyInfo repeatedOmKeyInfo =
(RepeatedOmKeyInfo) event.getValue();
- objectCountMap.computeIfPresent(countKey, (k, count) ->
- count > 0 ? count - repeatedOmKeyInfo.getOmKeyInfoList().size() :
0L);
+ objectCountMap.computeIfPresent(countKey, (k, count) -> count > 0 ?
+ count - repeatedOmKeyInfo.getOmKeyInfoList().size() : 0L);
Pair<Long, Long> result = repeatedOmKeyInfo.getTotalSize();
unreplicatedSizeCountMap.computeIfPresent(unReplicatedSizeKey,
(k, size) -> size > result.getLeft() ? size - result.getLeft() : 0L);
@@ -373,14 +378,37 @@ public class OmTableInsightTask implements ReconOmTask {
HashMap<String, Long> objectCountMap,
HashMap<String, Long>
unreplicatedSizeCountMap,
HashMap<String, Long> replicatedSizeCountMap)
{
+
+ if (event.getValue() != null) {
+ if (sizeRelatedTables.contains(tableName)) {
+ // Handle update for only size related tables
+ handleSizeRelatedTableUpdateEvent(event, tableName, objectCountMap,
+ unreplicatedSizeCountMap, replicatedSizeCountMap);
+ }
+ }
+ }
+
+
+ private void handleSizeRelatedTableUpdateEvent(
+ OMDBUpdateEvent<String, Object> event,
+ String tableName,
+ HashMap<String, Long> objectCountMap,
+ HashMap<String, Long> unreplicatedSizeCountMap,
+ HashMap<String, Long> replicatedSizeCountMap) {
+
+ if (event.getOldValue() == null) {
+ LOG.warn("Update event does not have the old Key Info for {}.",
+ event.getKey());
+ return;
+ }
String countKey = getTableCountKeyFromTable(tableName);
String unReplicatedSizeKey = getUnReplicatedSizeKeyFromTable(tableName);
String replicatedSizeKey = getReplicatedSizeKeyFromTable(tableName);
- // In Update event the count for the table will not change. So we don't
+
+ // In Update event the count for the open table will not change. So we
don't
// need to update the count. Except for RepeatedOmKeyInfo, for which the
// size of omKeyInfoList can change
- if (event.getValue() instanceof OmKeyInfo && event.getOldValue() != null &&
- sizeRelatedTables.contains(tableName)) {
+ if (event.getValue() instanceof OmKeyInfo) {
// Handle UPDATE for OpenKeyTable & OpenFileTable
OmKeyInfo oldKeyInfo = (OmKeyInfo) event.getOldValue();
OmKeyInfo newKeyInfo = (OmKeyInfo) event.getValue();
@@ -390,8 +418,7 @@ public class OmTableInsightTask implements ReconOmTask {
replicatedSizeCountMap.computeIfPresent(replicatedSizeKey,
(k, size) -> size - oldKeyInfo.getReplicatedSize() +
newKeyInfo.getReplicatedSize());
- } else if (event.getValue() instanceof RepeatedOmKeyInfo &&
- sizeRelatedTables.contains(tableName) && event.getOldValue() != null) {
+ } else if (event.getValue() instanceof RepeatedOmKeyInfo) {
// Handle UPDATE for DeletedTable
RepeatedOmKeyInfo oldRepeatedOmKeyInfo =
(RepeatedOmKeyInfo) event.getOldValue();
@@ -407,9 +434,6 @@ public class OmTableInsightTask implements ReconOmTask {
(k, size) -> size - oldSize.getLeft() + newSize.getLeft());
replicatedSizeCountMap.computeIfPresent(replicatedSizeKey,
(k, size) -> size - oldSize.getRight() + newSize.getRight());
- } else if (event.getValue() != null) {
- LOG.warn("Update event does not have the old Key Info for {}.",
- event.getKey());
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
