This is an automated email from the ASF dual-hosted git repository.
avijayan pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new b93bee5 AMBARI-23932 - Ambari Metrics reports incorrect values in
aggregated host metric data when requesting avg.
b93bee5 is described below
commit b93bee5beaa3b095b11700a70290ea415b895ec4
Author: Aravindan Vijayan <[email protected]>
AuthorDate: Tue May 22 14:36:29 2018 -0700
AMBARI-23932 - Ambari Metrics reports incorrect values in aggregated host
metric data when requesting avg.
---
.../core/timeline/PhoenixHBaseAccessor.java | 25 +++++++++++++++++-----
.../aggregators/TimelineMetricReadHelper.java | 8 ++++++-
.../core/timeline/query/PhoenixTransactSQL.java | 2 +-
3 files changed, 28 insertions(+), 7 deletions(-)
diff --git
a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
index f2a3bca..6f8a1de 100644
---
a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
+++
b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
@@ -38,6 +38,7 @@ import static
org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguratio
import static
org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HOST_AGGREGATOR_MINUTE_SLEEP_INTERVAL;
import static
org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HSTORE_COMPACTION_CLASS_KEY;
import static
org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HSTORE_ENGINE_CLASS;
+import static
org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_EVENT_METRIC_PATTERNS;
import static
org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TRANSIENT_METRIC_PATTERNS;
import static
org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HOST_DAILY_TABLE_TTL;
import static
org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HOST_HOUR_TABLE_TTL;
@@ -81,6 +82,7 @@ import static
org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.U
import static
org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.UPSERT_METRICS_SQL;
import static
org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.UPSERT_TRANSIENT_METRICS_SQL;
import static
org.apache.ambari.metrics.core.timeline.source.InternalSourceProvider.SOURCE_NAME.RAW_METRICS;
+import static
org.apache.hadoop.metrics2.sink.timeline.TimelineMetricUtils.getJavaMetricPatterns;
import java.io.IOException;
import java.sql.Connection;
@@ -195,6 +197,7 @@ public class PhoenixHBaseAccessor {
private final int cacheCommitInterval;
private final boolean skipBlockCacheForAggregatorsEnabled;
private TimelineMetricMetadataManager metadataManagerInstance;
+ private Set<String> eventMetricPatterns = new HashSet<>();
private Map<String, Integer> tableTTL = new HashMap<>();
@@ -240,6 +243,9 @@ public class PhoenixHBaseAccessor {
this.insertCache = new ArrayBlockingQueue<TimelineMetrics>(cacheSize);
this.skipBlockCacheForAggregatorsEnabled =
metricsConf.getBoolean(AGGREGATORS_SKIP_BLOCK_CACHE, false);
+ String eventMetricPatternStrings =
metricsConf.get(TIMELINE_METRICS_EVENT_METRIC_PATTERNS, StringUtils.EMPTY);
+
eventMetricPatterns.addAll(getJavaMetricPatterns(eventMetricPatternStrings));
+
tableTTL.put(METRICS_RECORD_TABLE_NAME,
metricsConf.getInt(PRECISION_TABLE_TTL, 1 * 86400)); // 1 day
tableTTL.put(CONTAINER_METRICS_TABLE_NAME,
metricsConf.getInt(CONTAINER_METRICS_TTL, 14 * 86400)); // 30 days
tableTTL.put(METRICS_AGGREGATE_MINUTE_TABLE_NAME,
metricsConf.getInt(HOST_MINUTE_TABLE_TTL, 7 * 86400)); //7 days
@@ -1037,10 +1043,10 @@ public class PhoenixHBaseAccessor {
}
for (Function f : functions) {
if (f.getReadFunction() == Function.ReadFunction.VALUE) {
- getTimelineMetricsFromResultSet(metrics, f, condition, rs);
+ getTimelineMetricsFromResultSet(metrics, f, condition, rs,
isEventDownsampledMetric(metricName));
} else {
SingleValuedTimelineMetric metric =
-
TIMELINE_METRIC_READ_HELPER.getAggregatedTimelineMetricFromResultSet(rs, f);
+
TIMELINE_METRIC_READ_HELPER.getAggregatedTimelineMetricFromResultSet(rs, f,
isEventDownsampledMetric(metricName));
if (condition.isGrouped()) {
metrics.addOrMergeTimelineMetric(metric);
@@ -1052,12 +1058,21 @@ public class PhoenixHBaseAccessor {
} else {
// No aggregation requested
// Execution never goes here, function always contain at least 1
element
- getTimelineMetricsFromResultSet(metrics, null, condition, rs);
+ getTimelineMetricsFromResultSet(metrics, null, condition, rs,
isEventDownsampledMetric(metricName));
+ }
+ }
+ }
+
+ private boolean isEventDownsampledMetric(String metricName) {
+ for (String pattern : eventMetricPatterns) {
+ if (metricName.matches(pattern)) {
+ return true;
}
}
+ return false;
}
- private void getTimelineMetricsFromResultSet(TimelineMetrics metrics,
Function f, Condition condition, ResultSet rs)
+ private void getTimelineMetricsFromResultSet(TimelineMetrics metrics,
Function f, Condition condition, ResultSet rs, boolean shouldSumAcrossTime)
throws SQLException, IOException {
if (condition.getPrecision().equals(Precision.SECONDS)) {
TimelineMetric metric =
TIMELINE_METRIC_READ_HELPER.getTimelineMetricFromResultSet(rs);
@@ -1072,7 +1087,7 @@ public class PhoenixHBaseAccessor {
} else {
SingleValuedTimelineMetric metric =
-
TIMELINE_METRIC_READ_HELPER.getAggregatedTimelineMetricFromResultSet(rs, f);
+
TIMELINE_METRIC_READ_HELPER.getAggregatedTimelineMetricFromResultSet(rs, f,
shouldSumAcrossTime);
if (condition.isGrouped()) {
metrics.addOrMergeTimelineMetric(metric);
} else {
diff --git
a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricReadHelper.java
b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricReadHelper.java
index 8975f44..541cb46 100644
---
a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricReadHelper.java
+++
b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricReadHelper.java
@@ -59,7 +59,8 @@ public class TimelineMetricReadHelper {
}
public SingleValuedTimelineMetric
getAggregatedTimelineMetricFromResultSet(ResultSet rs,
-
Function f) throws SQLException, IOException {
+
Function f,
+
boolean shouldSumMetricAcrossTime) throws SQLException, IOException {
byte[] uuid = rs.getBytes("UUID");
TimelineMetric timelineMetric =
metadataManagerInstance.getMetricFromUuid(uuid);
@@ -73,6 +74,8 @@ public class TimelineMetricReadHelper {
);
double value;
+ // GET request for sum & avg is handled as the same since 'summing' of
values across time does not make sense.
+ // If explicit sum downsampling is required across time, we have to use
ams-site : timeline.metrics.downsampler.event.metric.patterns.
switch(function.getReadFunction()){
case AVG:
value = rs.getDouble("METRIC_SUM") / rs.getInt("METRIC_COUNT");
@@ -85,6 +88,9 @@ public class TimelineMetricReadHelper {
break;
case SUM:
value = rs.getDouble("METRIC_SUM");
+ if (!shouldSumMetricAcrossTime) {
+ value = value / rs.getInt("METRIC_COUNT");
+ }
break;
default:
value = rs.getDouble("METRIC_SUM") / rs.getInt("METRIC_COUNT");
diff --git
a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/PhoenixTransactSQL.java
b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/PhoenixTransactSQL.java
index e0cc642..af24deb 100644
---
a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/PhoenixTransactSQL.java
+++
b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/PhoenixTransactSQL.java
@@ -379,7 +379,7 @@ public class PhoenixTransactSQL {
public static final String GET_AGGREGATED_HOST_METRIC_GROUPBY_SQL = "UPSERT
" +
"INTO %s (UUID, SERVER_TIME, METRIC_SUM, METRIC_COUNT, METRIC_MAX,
METRIC_MIN) " +
"SELECT UUID, %s AS SERVER_TIME, " +
- "ROUND(SUM(METRIC_SUM)/SUM(METRIC_COUNT),2), SUM(METRIC_COUNT),
MAX(METRIC_MAX), MIN(METRIC_MIN) " +
+ "SUM(METRIC_SUM), SUM(METRIC_COUNT), MAX(METRIC_MAX), MIN(METRIC_MIN) " +
"FROM %s WHERE%s SERVER_TIME > %s AND SERVER_TIME <= %s GROUP BY UUID";
/**
--
To stop receiving notification emails like this one, please contact
[email protected].
