lcspinter commented on a change in pull request #2916:
URL: https://github.com/apache/hive/pull/2916#discussion_r788981616
##########
File path:
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/metrics/DeltaFilesMetricReporter.java
##########
@@ -512,7 +260,168 @@ private void shutdown() {
}
}
- public static class DeltaFilesMetadata implements Serializable {
- public String dbName, tableName, partitionName;
+ public static void updateMetricsFromInitiator(AcidDirectory dir, String
dbName, String tableName, String partitionName,
+ Configuration conf, TxnStore txnHandler, long baseSize, Map<Path, Long>
deltaSizes) {
+ LOG.debug("Updating delta file metrics from initiator");
+ double deltaPctThreshold = MetastoreConf.getDoubleVar(conf,
MetastoreConf.ConfVars.METASTORE_DELTAMETRICS_DELTA_PCT_THRESHOLD);
+ int deltasThreshold = MetastoreConf.getIntVar(conf,
MetastoreConf.ConfVars.METASTORE_DELTAMETRICS_DELTA_NUM_THRESHOLD);
+ int obsoleteDeltasThreshold = MetastoreConf.getIntVar(conf,
+
MetastoreConf.ConfVars.METASTORE_DELTAMETRICS_OBSOLETE_DELTA_NUM_THRESHOLD);
+ try {
+ // We have an AcidDir from the initiator, therefore we can use that to
calculate active,small, obsolete delta
+ // count
+
+ int numDeltas = dir.getCurrentDirectories().size();
+ int numSmallDeltas = 0;
+
+ for (AcidUtils.ParsedDelta delta : dir.getCurrentDirectories()) {
+ if (deltaSizes.containsKey(delta.getPath())) {
+ long deltaSize = deltaSizes.get(delta.getPath());
+ if (baseSize != 0 && deltaSize / (float) baseSize <
deltaPctThreshold) {
+ numSmallDeltas++;
+ }
+ }
+ }
+
+ int numObsoleteDeltas =
filterOutBaseAndOriginalFiles(dir.getObsolete()).size();
+
+ if (numDeltas > deltasThreshold) {
+ updateMetrics(dbName, tableName, partitionName,
CompactionMetricsData.MetricType.NUM_DELTAS, numDeltas,
+ txnHandler);
+ }
+
+ if (numSmallDeltas > deltasThreshold) {
+ updateMetrics(dbName, tableName, partitionName,
CompactionMetricsData.MetricType.NUM_SMALL_DELTAS,
+ numSmallDeltas, txnHandler);
+ }
+
+ if (numObsoleteDeltas > obsoleteDeltasThreshold) {
+ updateMetrics(dbName, tableName, partitionName,
CompactionMetricsData.MetricType.NUM_OBSOLETE_DELTAS,
+ numObsoleteDeltas, txnHandler);
+ }
+
+ LOG.debug("Finished updating delta file metrics from initiator.\n
deltaPctThreshold = {}, deltasThreshold = {}, "
+ + "obsoleteDeltasThreshold = {}, numDeltas = {}, numSmallDeltas =
{}, numObsoleteDeltas = {}",
+ deltaPctThreshold, deltasThreshold, obsoleteDeltasThreshold,
numDeltas, numSmallDeltas, numObsoleteDeltas);
+
+ } catch (Throwable t) {
+ LOG.warn("Unknown throwable caught while updating delta metrics. Metrics
will not be updated.", t);
+ }
+ }
+
+ public static void updateMetricsFromWorker(AcidDirectory directory, String
dbName, String tableName, String partitionName,
+ CompactionType type, Configuration conf, IMetaStoreClient client) {
+ LOG.debug("Updating delta file metrics from worker");
+ int deltasThreshold = MetastoreConf.getIntVar(conf,
MetastoreConf.ConfVars.METASTORE_DELTAMETRICS_DELTA_NUM_THRESHOLD);
+ int obsoleteDeltasThreshold = MetastoreConf.getIntVar(conf,
+
MetastoreConf.ConfVars.METASTORE_DELTAMETRICS_OBSOLETE_DELTA_NUM_THRESHOLD);
+ try {
+ // we have an instance of the AcidDirectory before the compaction worker
was started
+ // from this we can get how many delta directories existed
+ // the previously active delta directories are now moved to obsolete
+ int numObsoleteDeltas = directory.getCurrentDirectories().size();
+ if (numObsoleteDeltas > obsoleteDeltasThreshold) {
+ updateMetrics(dbName, tableName, partitionName,
CompactionMetricsMetricType.NUM_OBSOLETE_DELTAS,
+ numObsoleteDeltas, client);
+ }
Review comment:
Corrected.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
