klcopp commented on a change in pull request #2493:
URL: https://github.com/apache/hive/pull/2493#discussion_r673965403
##########
File path:
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metrics/AcidMetricService.java
##########
@@ -38,7 +38,27 @@
import java.util.stream.Collectors;
import static
org.apache.hadoop.hive.metastore.HiveMetaStoreClient.MANUALLY_INITIATED_COMPACTION;
-import static org.apache.hadoop.hive.metastore.metrics.MetricsConstants.*;
Review comment:
+1
##########
File path:
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/metrics/DeltaFilesMetricReporter.java
##########
@@ -84,6 +84,8 @@
public static final String OBJECT_NAME_PREFIX =
"metrics:type=compaction,name=";
+ private static long lastSuccessfulLoggingTime = 0;
Review comment:
This should probably be initialized to System.currentTimeMillis();
##########
File path:
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metrics/AcidMetricService.java
##########
@@ -52,6 +72,7 @@
private Configuration conf;
private TxnStore txnHandler;
+ private long lastSuccessfulLoggingTime = 0;
Review comment:
This should probably be initialized to System.currentTimeMillis();
##########
File path:
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/metrics/DeltaFilesMetricReporter.java
##########
@@ -212,11 +215,41 @@ public static void mergeDeltaFilesStats(AcidDirectory
dir, long checkThresholdIn
}
}
}
+
+ logDeltaDirMetrics(dir, conf, numObsoleteDeltas, numDeltas,
numSmallDeltas);
+
String path = getRelPath(dir);
newDeltaFilesStats(numObsoleteDeltas, numDeltas, numSmallDeltas)
.forEach((type, cnt) -> deltaFilesStats.computeIfAbsent(type, v -> new
HashMap<>()).put(path, cnt));
}
+ private static void logDeltaDirMetrics(AcidDirectory dir, Configuration
conf, int numObsoleteDeltas, int numDeltas,
+ int numSmallDeltas) {
+ long loggerFrequency = HiveConf
+ .getTimeVar(conf,
HiveConf.ConfVars.HIVE_COMPACTOR_ACID_METRICS_LOGGER_FREQUENCY,
TimeUnit.MILLISECONDS);
+ if (loggerFrequency <= 0) {
+ return;
+ }
+ long currentTime = System.currentTimeMillis();
+ if (lastSuccessfulLoggingTime == 0 || currentTime >=
lastSuccessfulLoggingTime + loggerFrequency) {
Review comment:
I think the first part should be: loggerFrequency == 0
##########
File path:
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
##########
@@ -295,11 +295,12 @@
"SELECT COUNT(*), MIN(\"TXN_ID\"), ({0} - MIN(\"TXN_STARTED\"))/1000
FROM \"TXNS\" WHERE \"TXN_STATE\"='" +
TxnStatus.ABORTED + "') \"A\" CROSS JOIN (" +
"SELECT COUNT(*), ({0} - MIN(\"HL_ACQUIRED_AT\"))/1000 FROM
\"HIVE_LOCKS\") \"HL\" CROSS JOIN (" +
- "SELECT COUNT(*) FROM (SELECT COUNT(\"TXN_ID\"), \"TC_DATABASE\",
\"TC_TABLE\", \"TC_PARTITION\" FROM \"TXN_COMPONENTS\" " +
- "INNER JOIN \"TXNS\" ON \"TC_TXNID\" = \"TXN_ID\" WHERE
\"TXN_STATE\"='" + TxnStatus.ABORTED + "' " +
- "GROUP BY \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\" HAVING
COUNT(\"TXN_ID\") > ?) \"L\") \"L\" CROSS JOIN (" +
"SELECT ({0} - MIN(\"CQ_COMMIT_TIME\"))/1000 from \"COMPACTION_QUEUE\"
WHERE " +
"\"CQ_STATE\"=''" + Character.toString(READY_FOR_CLEANING) + "'')
OLDEST_CLEAN";
+ private static final String SELECT_TABLES_WITH_X_ABORTED_TXNS =
+ "SELECT \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\" FROM
\"TXN_COMPONENTS\" " +
+ "INNER JOIN \"TXNS\" ON \"TC_TXNID\" = \"TXN_ID\" WHERE
\"TXN_STATE\" = " + TxnStatus.ABORTED +
Review comment:
Probably the issue here is that TxnStatus.ABORTED needs to be surrounded
by single quotes
##########
File path:
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metrics/AcidMetricService.java
##########
@@ -85,36 +106,134 @@ public void run() {
private void collectMetrics() throws MetaException {
ShowCompactResponse currentCompactions = txnHandler.showCompact(new
ShowCompactRequest());
- updateMetricsFromShowCompact(currentCompactions);
+ updateMetricsFromShowCompact(currentCompactions, conf);
updateDBMetrics();
}
private void updateDBMetrics() throws MetaException {
MetricsInfo metrics = txnHandler.getMetricsInfo();
Metrics.getOrCreateGauge(NUM_TXN_TO_WRITEID).set(metrics.getTxnToWriteIdCount());
+ logDbMetrics(metrics);
Metrics.getOrCreateGauge(NUM_COMPLETED_TXN_COMPONENTS).set(metrics.getCompletedTxnsCount());
-
Metrics.getOrCreateGauge(NUM_OPEN_REPL_TXNS).set(metrics.getOpenReplTxnsCount());
Metrics.getOrCreateGauge(OLDEST_OPEN_REPL_TXN_ID).set(metrics.getOldestOpenReplTxnId());
Metrics.getOrCreateGauge(OLDEST_OPEN_REPL_TXN_AGE).set(metrics.getOldestOpenReplTxnAge());
Metrics.getOrCreateGauge(NUM_OPEN_NON_REPL_TXNS).set(metrics.getOpenNonReplTxnsCount());
Metrics.getOrCreateGauge(OLDEST_OPEN_NON_REPL_TXN_ID).set(metrics.getOldestOpenNonReplTxnId());
Metrics.getOrCreateGauge(OLDEST_OPEN_NON_REPL_TXN_AGE).set(metrics.getOldestOpenNonReplTxnAge());
-
Metrics.getOrCreateGauge(NUM_ABORTED_TXNS).set(metrics.getAbortedTxnsCount());
Metrics.getOrCreateGauge(OLDEST_ABORTED_TXN_ID).set(metrics.getOldestAbortedTxnId());
Metrics.getOrCreateGauge(OLDEST_ABORTED_TXN_AGE).set(metrics.getOldestAbortedTxnAge());
-
Metrics.getOrCreateGauge(NUM_LOCKS).set(metrics.getLocksCount());
Metrics.getOrCreateGauge(OLDEST_LOCK_AGE).set(metrics.getOldestLockAge());
+
Metrics.getOrCreateGauge(TABLES_WITH_X_ABORTED_TXNS).set(metrics.getTablesWithXAbortedTxnsCount());
+
Metrics.getOrCreateGauge(OLDEST_READY_FOR_CLEANING_AGE).set(metrics.getOldestReadyForCleaningAge());
+ }
+
+ private void logDbMetrics(MetricsInfo metrics) {
+ long loggingFrequency = MetastoreConf
+ .getTimeVar(conf,
MetastoreConf.ConfVars.COMPACTOR_ACID_METRICS_LOGGER_FREQUENCY,
TimeUnit.MILLISECONDS);
+ if (loggingFrequency <= 0) {
+ return;
+ }
+ long currentTime = System.currentTimeMillis();
+ if (lastSuccessfulLoggingTime == 0 || currentTime >=
lastSuccessfulLoggingTime + loggingFrequency) {
Review comment:
I think the first part should be: loggingFrequency == 0
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
