This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 1789489c67e HIVE-26471: Addendum: Fix metric computing (Laszlo Vegh,
reviewed by Denys Kuzmenko)
1789489c67e is described below
commit 1789489c67e5131f3a444cf7b9ac039f276ab6c4
Author: veghlaci05 <[email protected]>
AuthorDate: Tue Oct 11 16:46:29 2022 +0200
HIVE-26471: Addendum: Fix metric computing (Laszlo Vegh, reviewed by Denys
Kuzmenko)
Closes #3645
---
.../hive/metastore/metrics/AcidMetricService.java | 10 +++-
.../metastore/metrics/CompactionMetricData.java | 55 +++++++++++++++++-----
.../hive/metastore/metrics/MetricsConstants.java | 5 +-
3 files changed, 56 insertions(+), 14 deletions(-)
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metrics/AcidMetricService.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metrics/AcidMetricService.java
index 8fa81d1c4a4..cc29af053f2 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metrics/AcidMetricService.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metrics/AcidMetricService.java
@@ -58,7 +58,10 @@ import static
org.apache.hadoop.hive.metastore.metrics.MetricsConstants.COMPACTI
import static
org.apache.hadoop.hive.metastore.metrics.MetricsConstants.COMPACTION_OLDEST_CLEANING_AGE;
import static
org.apache.hadoop.hive.metastore.metrics.MetricsConstants.COMPACTION_OLDEST_ENQUEUE_AGE;
import static
org.apache.hadoop.hive.metastore.metrics.MetricsConstants.COMPACTION_OLDEST_WORKING_AGE;
-import static
org.apache.hadoop.hive.metastore.metrics.MetricsConstants.COMPACTION_POOLS_ITEM_COUNT;
+import static
org.apache.hadoop.hive.metastore.metrics.MetricsConstants.COMPACTION_POOLS_INITIATED_ITEM_COUNT;
+import static
org.apache.hadoop.hive.metastore.metrics.MetricsConstants.COMPACTION_POOLS_OLDEST_INITIATED_AGE;
+import static
org.apache.hadoop.hive.metastore.metrics.MetricsConstants.COMPACTION_POOLS_OLDEST_WORKING_AGE;
+import static
org.apache.hadoop.hive.metastore.metrics.MetricsConstants.COMPACTION_POOLS_WORKING_ITEM_COUNT;
import static
org.apache.hadoop.hive.metastore.metrics.MetricsConstants.COMPACTION_STATUS_PREFIX;
import static
org.apache.hadoop.hive.metastore.metrics.MetricsConstants.NUM_ABORTED_TXNS;
import static
org.apache.hadoop.hive.metastore.metrics.MetricsConstants.NUM_COMPLETED_TXN_COMPONENTS;
@@ -333,7 +336,10 @@ public class AcidMetricService implements
MetastoreTaskThread {
}
}
-
Metrics.getOrCreateMapMetrics(COMPACTION_POOLS_ITEM_COUNT).update(metricData.getPoolCount());
+
Metrics.getOrCreateMapMetrics(COMPACTION_POOLS_INITIATED_ITEM_COUNT).update(metricData.getInitiatedCountPerPool());
+
Metrics.getOrCreateMapMetrics(COMPACTION_POOLS_WORKING_ITEM_COUNT).update(metricData.getWorkingCountPerPool());
+
Metrics.getOrCreateMapMetrics(COMPACTION_POOLS_OLDEST_INITIATED_AGE).update(metricData.getLongestEnqueueDurationPerPool());
+
Metrics.getOrCreateMapMetrics(COMPACTION_POOLS_OLDEST_WORKING_AGE).update(metricData.getLongestWorkingDurationPerPool());
updateOldestCompactionMetric(COMPACTION_OLDEST_ENQUEUE_AGE,
metricData.getOldestEnqueueTime());
updateOldestCompactionMetric(COMPACTION_OLDEST_WORKING_AGE,
metricData.getOldestWorkingTime());
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metrics/CompactionMetricData.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metrics/CompactionMetricData.java
index 253e36553b3..964111680e1 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metrics/CompactionMetricData.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metrics/CompactionMetricData.java
@@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableList;
import org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement;
import org.apache.hadoop.hive.metastore.txn.TxnStore;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -45,7 +46,10 @@ final class CompactionMetricData {
private Map<String, Long> stateCount;
- private Map<String, Integer> poolCount;
+ private Map<String, Integer> initiatedCountPerPool;
+ private Map<String, Integer> workingCountPerPool;
+ private Map<String, Integer> longestEnqueueDurationPerPool;
+ private Map<String, Integer> longestWorkingDurationPerPool;
private Double failedCompactionPercentage;
@@ -67,11 +71,16 @@ final class CompactionMetricData {
private void init() {
final Map<String, ShowCompactResponseElement> lastElements = new
HashMap<>();
- poolCount = new HashMap<>();
+ initiatedCountPerPool = new HashMap<>();
+ workingCountPerPool = new HashMap<>();
+ longestEnqueueDurationPerPool = new HashMap<>();
+ longestWorkingDurationPerPool = new HashMap<>();
oldestEnqueueTime = OLDEST_TIME_NO_VALUE;
oldestWorkingTime = OLDEST_TIME_NO_VALUE;
oldestCleaningTime = OLDEST_TIME_NO_VALUE;
+
+ long currentTime = System.currentTimeMillis();
for (ShowCompactResponseElement element : compacts) {
final String key = element.getDbname() + "/" + element.getTablename() +
(element.getPartitionname() != null ? "/" +
element.getPartitionname() : "");
@@ -79,16 +88,28 @@ final class CompactionMetricData {
// If new key, add the element, if there is an existing one, change to
the element if the element.id is greater than old.id
lastElements.compute(key, (k, old) -> (old == null) ? element :
(element.getId() > old.getId() ? element : old));
- // find the oldest elements with initiated and working states
String state = element.getState();
- if (TxnStore.INITIATED_RESPONSE.equals(state) && (oldestEnqueueTime >
element.getEnqueueTime())) {
- oldestEnqueueTime = element.getEnqueueTime();
- poolCount.compute(element.getPoolName(), (k, old) -> (old == null) ? 1
: old + 1);
+ if (TxnStore.INITIATED_RESPONSE.equals(state)) {
+ final int enqueueSeconds = (int) ((currentTime -
element.getEnqueueTime()) / 1000);
+ longestEnqueueDurationPerPool.compute(element.getPoolName(), (k, old)
-> (old == null) ? enqueueSeconds : Math.max(enqueueSeconds, old));
+
+ initiatedCountPerPool.compute(element.getPoolName(), (k, old) -> (old
== null) ? 1 : old + 1);
+ // find the oldest element
+ if (oldestEnqueueTime > element.getEnqueueTime()) {
+ oldestEnqueueTime = element.getEnqueueTime();
+ }
}
if (element.isSetStart()) {
- if (TxnStore.WORKING_RESPONSE.equals(state) && (oldestWorkingTime >
element.getStart())) {
- oldestWorkingTime = element.getStart();
+ if (TxnStore.WORKING_RESPONSE.equals(state)) {
+ final int startSeconds = (int) ((currentTime - element.getStart()) /
1000);
+ longestWorkingDurationPerPool.compute(element.getPoolName(), (k,
old) -> (old == null) ? startSeconds : Math.max(startSeconds, old));
+
+ workingCountPerPool.compute(element.getPoolName(), (k, old) -> (old
== null) ? 1 : old + 1);
+ // find the oldest element
+ if (oldestWorkingTime > element.getStart()) {
+ oldestWorkingTime = element.getStart();
+ }
}
}
@@ -149,11 +170,23 @@ final class CompactionMetricData {
}
Map<String, Long> getStateCount() {
- return new HashMap<>(stateCount);
+ return Collections.unmodifiableMap(stateCount);
+ }
+
+ public Map<String, Integer> getInitiatedCountPerPool() {
+ return Collections.unmodifiableMap(initiatedCountPerPool);
+ }
+
+ public Map<String, Integer> getWorkingCountPerPool() {
+ return Collections.unmodifiableMap(workingCountPerPool);
+ }
+
+ public Map<String, Integer> getLongestEnqueueDurationPerPool() {
+ return Collections.unmodifiableMap(longestEnqueueDurationPerPool);
}
- public Map<String, Integer> getPoolCount() {
- return new HashMap<>(poolCount);
+ public Map<String, Integer> getLongestWorkingDurationPerPool() {
+ return Collections.unmodifiableMap(longestWorkingDurationPerPool);
}
Long getOldestEnqueueTime() {
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metrics/MetricsConstants.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metrics/MetricsConstants.java
index 8d3037d263c..46512ab3c4b 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metrics/MetricsConstants.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metrics/MetricsConstants.java
@@ -31,7 +31,10 @@ public class MetricsConstants {
public static final String COMPACTION_CLEANER_CYCLE_DURATION =
"compaction_cleaner_cycle_duration";
public static final String COMPACTION_CLEANER_FAILURE_COUNTER =
"compaction_cleaner_failure_counter";
public static final String COMPACTION_WORKER_CYCLE =
"compaction_worker_cycle";
- public static final String COMPACTION_POOLS_ITEM_COUNT =
"compaction_pools_item_count";
+ public static final String COMPACTION_POOLS_INITIATED_ITEM_COUNT =
"compaction_pools_initiated_item_count";
+ public static final String COMPACTION_POOLS_WORKING_ITEM_COUNT =
"compaction_pools_working_item_count";
+ public static final String COMPACTION_POOLS_OLDEST_INITIATED_AGE =
"compaction_pools_oldest_enqueue_age_in_sec";
+ public static final String COMPACTION_POOLS_OLDEST_WORKING_AGE =
"compaction_pools_oldest_working_age_in_sec";
public static final String OLDEST_OPEN_REPL_TXN_ID =
"oldest_open_repl_txn_id";
public static final String OLDEST_OPEN_NON_REPL_TXN_ID =
"oldest_open_non_repl_txn_id";
