[
https://issues.apache.org/jira/browse/HIVE-25345?focusedWorklogId=626135&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-626135
]
ASF GitHub Bot logged work on HIVE-25345:
-----------------------------------------
Author: ASF GitHub Bot
Created on: 21/Jul/21 13:33
Start Date: 21/Jul/21 13:33
Worklog Time Spent: 10m
Work Description: klcopp commented on a change in pull request #2493:
URL: https://github.com/apache/hive/pull/2493#discussion_r673965403
##########
File path:
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metrics/AcidMetricService.java
##########
@@ -38,7 +38,27 @@
import java.util.stream.Collectors;
import static
org.apache.hadoop.hive.metastore.HiveMetaStoreClient.MANUALLY_INITIATED_COMPACTION;
-import static org.apache.hadoop.hive.metastore.metrics.MetricsConstants.*;
Review comment:
+1
##########
File path:
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/metrics/DeltaFilesMetricReporter.java
##########
@@ -84,6 +84,8 @@
public static final String OBJECT_NAME_PREFIX =
"metrics:type=compaction,name=";
+ private static long lastSuccessfulLoggingTime = 0;
Review comment:
This should probably be initialized to System.currentTimeMillis();
##########
File path:
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metrics/AcidMetricService.java
##########
@@ -52,6 +72,7 @@
private Configuration conf;
private TxnStore txnHandler;
+ private long lastSuccessfulLoggingTime = 0;
Review comment:
This should probably be initialized to System.currentTimeMillis();
##########
File path:
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/metrics/DeltaFilesMetricReporter.java
##########
@@ -212,11 +215,41 @@ public static void mergeDeltaFilesStats(AcidDirectory
dir, long checkThresholdIn
}
}
}
+
+ logDeltaDirMetrics(dir, conf, numObsoleteDeltas, numDeltas,
numSmallDeltas);
+
String path = getRelPath(dir);
newDeltaFilesStats(numObsoleteDeltas, numDeltas, numSmallDeltas)
.forEach((type, cnt) -> deltaFilesStats.computeIfAbsent(type, v -> new
HashMap<>()).put(path, cnt));
}
+ private static void logDeltaDirMetrics(AcidDirectory dir, Configuration
conf, int numObsoleteDeltas, int numDeltas,
+ int numSmallDeltas) {
+ long loggerFrequency = HiveConf
+ .getTimeVar(conf,
HiveConf.ConfVars.HIVE_COMPACTOR_ACID_METRICS_LOGGER_FREQUENCY,
TimeUnit.MILLISECONDS);
+ if (loggerFrequency <= 0) {
+ return;
+ }
+ long currentTime = System.currentTimeMillis();
+ if (lastSuccessfulLoggingTime == 0 || currentTime >=
lastSuccessfulLoggingTime + loggerFrequency) {
Review comment:
I think the first part should be: loggerFrequency == 0
##########
File path:
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
##########
@@ -295,11 +295,12 @@
"SELECT COUNT(*), MIN(\"TXN_ID\"), ({0} - MIN(\"TXN_STARTED\"))/1000
FROM \"TXNS\" WHERE \"TXN_STATE\"='" +
TxnStatus.ABORTED + "') \"A\" CROSS JOIN (" +
"SELECT COUNT(*), ({0} - MIN(\"HL_ACQUIRED_AT\"))/1000 FROM
\"HIVE_LOCKS\") \"HL\" CROSS JOIN (" +
- "SELECT COUNT(*) FROM (SELECT COUNT(\"TXN_ID\"), \"TC_DATABASE\",
\"TC_TABLE\", \"TC_PARTITION\" FROM \"TXN_COMPONENTS\" " +
- "INNER JOIN \"TXNS\" ON \"TC_TXNID\" = \"TXN_ID\" WHERE
\"TXN_STATE\"='" + TxnStatus.ABORTED + "' " +
- "GROUP BY \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\" HAVING
COUNT(\"TXN_ID\") > ?) \"L\") \"L\" CROSS JOIN (" +
"SELECT ({0} - MIN(\"CQ_COMMIT_TIME\"))/1000 from \"COMPACTION_QUEUE\"
WHERE " +
"\"CQ_STATE\"=''" + Character.toString(READY_FOR_CLEANING) + "'')
OLDEST_CLEAN";
+ private static final String SELECT_TABLES_WITH_X_ABORTED_TXNS =
+ "SELECT \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\" FROM
\"TXN_COMPONENTS\" " +
+ "INNER JOIN \"TXNS\" ON \"TC_TXNID\" = \"TXN_ID\" WHERE
\"TXN_STATE\" = " + TxnStatus.ABORTED +
Review comment:
Probably the issue here is that TxnStatus.ABORTED needs to be surrounded
by single quotes
##########
File path:
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metrics/AcidMetricService.java
##########
@@ -85,36 +106,134 @@ public void run() {
private void collectMetrics() throws MetaException {
ShowCompactResponse currentCompactions = txnHandler.showCompact(new
ShowCompactRequest());
- updateMetricsFromShowCompact(currentCompactions);
+ updateMetricsFromShowCompact(currentCompactions, conf);
updateDBMetrics();
}
private void updateDBMetrics() throws MetaException {
MetricsInfo metrics = txnHandler.getMetricsInfo();
Metrics.getOrCreateGauge(NUM_TXN_TO_WRITEID).set(metrics.getTxnToWriteIdCount());
+ logDbMetrics(metrics);
Metrics.getOrCreateGauge(NUM_COMPLETED_TXN_COMPONENTS).set(metrics.getCompletedTxnsCount());
-
Metrics.getOrCreateGauge(NUM_OPEN_REPL_TXNS).set(metrics.getOpenReplTxnsCount());
Metrics.getOrCreateGauge(OLDEST_OPEN_REPL_TXN_ID).set(metrics.getOldestOpenReplTxnId());
Metrics.getOrCreateGauge(OLDEST_OPEN_REPL_TXN_AGE).set(metrics.getOldestOpenReplTxnAge());
Metrics.getOrCreateGauge(NUM_OPEN_NON_REPL_TXNS).set(metrics.getOpenNonReplTxnsCount());
Metrics.getOrCreateGauge(OLDEST_OPEN_NON_REPL_TXN_ID).set(metrics.getOldestOpenNonReplTxnId());
Metrics.getOrCreateGauge(OLDEST_OPEN_NON_REPL_TXN_AGE).set(metrics.getOldestOpenNonReplTxnAge());
-
Metrics.getOrCreateGauge(NUM_ABORTED_TXNS).set(metrics.getAbortedTxnsCount());
Metrics.getOrCreateGauge(OLDEST_ABORTED_TXN_ID).set(metrics.getOldestAbortedTxnId());
Metrics.getOrCreateGauge(OLDEST_ABORTED_TXN_AGE).set(metrics.getOldestAbortedTxnAge());
-
Metrics.getOrCreateGauge(NUM_LOCKS).set(metrics.getLocksCount());
Metrics.getOrCreateGauge(OLDEST_LOCK_AGE).set(metrics.getOldestLockAge());
+
Metrics.getOrCreateGauge(TABLES_WITH_X_ABORTED_TXNS).set(metrics.getTablesWithXAbortedTxnsCount());
+
Metrics.getOrCreateGauge(OLDEST_READY_FOR_CLEANING_AGE).set(metrics.getOldestReadyForCleaningAge());
+ }
+
+ private void logDbMetrics(MetricsInfo metrics) {
+ long loggingFrequency = MetastoreConf
+ .getTimeVar(conf,
MetastoreConf.ConfVars.COMPACTOR_ACID_METRICS_LOGGER_FREQUENCY,
TimeUnit.MILLISECONDS);
+ if (loggingFrequency <= 0) {
+ return;
+ }
+ long currentTime = System.currentTimeMillis();
+ if (lastSuccessfulLoggingTime == 0 || currentTime >=
lastSuccessfulLoggingTime + loggingFrequency) {
Review comment:
I think the first part should be: loggingFrequency == 0
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: gitbox-unsubscr...@hive.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
-------------------
Worklog Id: (was: 626135)
Time Spent: 1h 20m (was: 1h 10m)
> Add logging based on new compaction metrics
> -------------------------------------------
>
> Key: HIVE-25345
> URL: https://issues.apache.org/jira/browse/HIVE-25345
> Project: Hive
> Issue Type: Improvement
> Reporter: László Pintér
> Assignee: László Pintér
> Priority: Major
> Labels: pull-request-available
> Time Spent: 1h 20m
> Remaining Estimate: 0h
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
