Repository: ambari Updated Branches: refs/heads/trunk 5a43202b4 -> c14025873
Revert "AMBARI-8973 Add precision option support to Ambari Metrics API (dsen)" This reverts commit e7e865dee2f2305e9723f21ad80aa723cff47fc0. Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c1402587 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c1402587 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c1402587 Branch: refs/heads/trunk Commit: c1402587369a508942ff060ce336bf7b2c7e1bc4 Parents: 5a43202 Author: Dmytro Sen <[email protected]> Authored: Mon Jan 12 15:51:03 2015 +0200 Committer: Dmytro Sen <[email protected]> Committed: Mon Jan 12 15:51:59 2015 +0200 ---------------------------------------------------------------------- .../timeline/HBaseTimelineMetricStore.java | 8 +- .../metrics/timeline/PhoenixHBaseAccessor.java | 48 +--- .../metrics/timeline/PhoenixTransactSQL.java | 112 +------- .../metrics/timeline/Precision.java | 37 --- .../timeline/TimelineMetricAggregator.java | 2 +- .../TimelineMetricClusterAggregator.java | 2 +- .../TimelineMetricClusterAggregatorHourly.java | 6 +- .../metrics/timeline/TimelineMetricStore.java | 5 +- .../webapp/TimelineWebServices.java | 19 +- .../timeline/AbstractMiniHBaseClusterTest.java | 26 +- .../metrics/timeline/ITClusterAggregator.java | 121 +++++++-- .../metrics/timeline/ITMetricAggregator.java | 60 +++- .../timeline/ITPhoenixHBaseAccessor.java | 271 ------------------- .../metrics/timeline/MetricTestHelper.java | 96 ------- .../metrics/timeline/TestClusterSuite.java | 2 +- .../timeline/TestPhoenixTransactSQL.java | 133 +-------- .../timeline/TestTimelineMetricStore.java | 4 +- .../webapp/TestTimelineWebServices.java | 2 +- .../internal/AbstractProviderModule.java | 4 +- .../metrics/timeline/AMSPropertyProvider.java | 3 +- 20 files changed, 193 insertions(+), 768 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c1402587/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java index 1a0f530..3238312 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java @@ -113,11 +113,11 @@ public class HBaseTimelineMetricStore extends AbstractService @Override public TimelineMetrics getTimelineMetrics(List<String> metricNames, String hostname, String applicationId, String instanceId, - Long startTime, Long endTime, Precision precision, Integer limit, + Long startTime, Long endTime, Integer limit, boolean groupedByHosts) throws SQLException, IOException { Condition condition = new LikeCondition(metricNames, hostname, applicationId, - instanceId, startTime, endTime, precision, limit, groupedByHosts); + instanceId, startTime, endTime, limit, groupedByHosts); if (hostname == null) { return hBaseAccessor.getAggregateMetricRecords(condition); @@ -129,12 +129,12 @@ public class HBaseTimelineMetricStore extends AbstractService @Override public TimelineMetric getTimelineMetric(String metricName, String hostname, String applicationId, String instanceId, Long startTime, - Long endTime, Precision precision, Integer limit) + Long endTime, Integer limit) throws SQLException, IOException { TimelineMetrics metrics = hBaseAccessor.getMetricRecords( new LikeCondition(Collections.singletonList(metricName), hostname, - applicationId, instanceId, startTime, endTime, precision, limit, true) + applicationId, instanceId, startTime, endTime, limit, true) ); TimelineMetric metric = new TimelineMetric(); http://git-wip-us.apache.org/repos/asf/ambari/blob/c1402587/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java index 11ada0c..8be5112 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java @@ -181,23 +181,6 @@ public class PhoenixHBaseAccessor { return metric; } - static TimelineMetric getAggregatedTimelineMetricFromResultSet(ResultSet rs) - throws SQLException, IOException { - TimelineMetric metric = new TimelineMetric(); - metric.setMetricName(rs.getString("METRIC_NAME")); - metric.setHostName(rs.getString("HOSTNAME")); - metric.setAppId(rs.getString("APP_ID")); - metric.setInstanceId(rs.getString("INSTANCE_ID")); - metric.setTimestamp(rs.getLong("SERVER_TIME")); - metric.setStartTime(rs.getLong("SERVER_TIME")); - metric.setType(rs.getString("UNITS")); - Map<Long, Double> valueMap = new HashMap<Long, Double>(); - valueMap.put(rs.getLong("SERVER_TIME"), - rs.getDouble("METRIC_SUM") / rs.getInt("METRIC_COUNT")); - metric.setMetricValues(valueMap); - return metric; - } - private static Map<Long, Double> readLastMetricValueFromJSON(String json) throws IOException { Map<Long, Double> values = readMetricFromJSON(json); @@ -430,13 +413,7 @@ public class PhoenixHBaseAccessor { ResultSet rs = stmt.executeQuery(); while (rs.next()) { - TimelineMetric metric; - if (condition.getPrecision() == Precision.HOURS - || condition.getPrecision() == Precision.MINUTES) { - metric = getAggregatedTimelineMetricFromResultSet(rs); - } else { - metric = getTimelineMetricFromResultSet(rs); - } + TimelineMetric metric = getTimelineMetricFromResultSet(rs); if (condition.isGrouped()) { metrics.addOrMergeTimelineMetric(metric); @@ -514,12 +491,7 @@ public class PhoenixHBaseAccessor { ResultSet rs = stmt.executeQuery(); while (rs.next()) { - TimelineMetric metric; - if (condition.getPrecision() == Precision.HOURS) { - metric = getAggregateHoursTimelineMetricFromResultSet(rs); - } else { - metric = getAggregateTimelineMetricFromResultSet(rs); - } + TimelineMetric metric = getAggregateTimelineMetricFromResultSet(rs); if (condition.isGrouped()) { metrics.addOrMergeTimelineMetric(metric); @@ -588,22 +560,6 @@ public class PhoenixHBaseAccessor { return metric; } - private TimelineMetric getAggregateHoursTimelineMetricFromResultSet( - ResultSet rs) throws SQLException { - TimelineMetric metric = new TimelineMetric(); - metric.setMetricName(rs.getString("METRIC_NAME")); - metric.setAppId(rs.getString("APP_ID")); - metric.setInstanceId(rs.getString("INSTANCE_ID")); - metric.setTimestamp(rs.getLong("SERVER_TIME")); - metric.setStartTime(rs.getLong("SERVER_TIME")); - Map<Long, Double> valueMap = new HashMap<Long, Double>(); - valueMap.put(rs.getLong("SERVER_TIME"), - rs.getDouble("METRIC_SUM") / rs.getInt("METRIC_COUNT")); - metric.setMetricValues(valueMap); - - return metric; - } - private void verifyCondition(Condition condition) throws SQLException { if (condition.isEmpty()) { throw new SQLException("No filter criteria specified."); http://git-wip-us.apache.org/repos/asf/ambari/blob/c1402587/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixTransactSQL.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixTransactSQL.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixTransactSQL.java index df6da31..421d533 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixTransactSQL.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixTransactSQL.java @@ -192,17 +192,7 @@ public class PhoenixTransactSQL { "HOSTS_COUNT, " + "METRIC_MAX, " + "METRIC_MIN " + - "FROM %s"; - - public static final String GET_CLUSTER_AGGREGATE_HOURLY_SQL = "SELECT %s " + - "METRIC_NAME, APP_ID, " + - "INSTANCE_ID, SERVER_TIME, " + - "UNITS, " + - "METRIC_SUM, " + - "METRIC_COUNT, " + - "METRIC_MAX, " + - "METRIC_MIN " + - "FROM %s"; + "FROM METRIC_AGGREGATE"; public static final String METRICS_RECORD_TABLE_NAME = "METRIC_RECORD"; public static final String METRICS_AGGREGATE_MINUTE_TABLE_NAME = @@ -216,8 +206,6 @@ public class PhoenixTransactSQL { public static final String DEFAULT_TABLE_COMPRESSION = "SNAPPY"; public static final String DEFAULT_ENCODING = "FAST_DIFF"; public static final long NATIVE_TIME_RANGE_DELTA = 120000; // 2 minutes - public static final long HOUR = 3600000; // 1 hour - public static final long DAY = 86400000; // 1 day /** * Filter to optimize HBase scan by using file timestamps. This prevents @@ -239,43 +227,9 @@ public class PhoenixTransactSQL { if (condition.getStatement() != null) { stmtStr = condition.getStatement(); } else { - - String metricsTable; - String query; - if (condition.getPrecision() == null) { - Long timeRange = condition.getEndTime() - condition.getStartTime(); - if (timeRange > DAY) { - metricsTable = METRICS_AGGREGATE_HOURLY_TABLE_NAME; - query = GET_METRIC_AGGREGATE_ONLY_SQL; - condition.setPrecision(Precision.HOURS); - } else if (timeRange > HOUR) { - metricsTable = METRICS_AGGREGATE_MINUTE_TABLE_NAME; - query = GET_METRIC_AGGREGATE_ONLY_SQL; - condition.setPrecision(Precision.MINUTES); - } else { - metricsTable = METRICS_RECORD_TABLE_NAME; - query = GET_METRIC_SQL; - condition.setPrecision(Precision.SECONDS); - } - } else { - switch (condition.getPrecision()) { - case HOURS: - metricsTable = METRICS_AGGREGATE_HOURLY_TABLE_NAME; - query = GET_METRIC_AGGREGATE_ONLY_SQL; - break; - case MINUTES: - metricsTable = METRICS_AGGREGATE_MINUTE_TABLE_NAME; - query = GET_METRIC_AGGREGATE_ONLY_SQL; - break; - default: - metricsTable = METRICS_RECORD_TABLE_NAME; - query = GET_METRIC_SQL; - } - } - - stmtStr = String.format(query, + stmtStr = String.format(GET_METRIC_SQL, getNaiveTimeRangeHint(condition.getStartTime(), NATIVE_TIME_RANGE_DELTA), - metricsTable); + METRICS_RECORD_TABLE_NAME); } StringBuilder sb = new StringBuilder(stmtStr); @@ -424,32 +378,7 @@ public class PhoenixTransactSQL { throw new IllegalArgumentException("Condition is empty."); } - String metricsAggregateTable; - String queryStmt; - if (condition.getPrecision() == null) { - Long timeRange = condition.getEndTime() - condition.getStartTime(); - if (timeRange > DAY) { - metricsAggregateTable = METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME; - queryStmt = GET_CLUSTER_AGGREGATE_HOURLY_SQL; - condition.setPrecision(Precision.HOURS); - } else { - metricsAggregateTable = METRICS_CLUSTER_AGGREGATE_TABLE_NAME; - queryStmt = GET_CLUSTER_AGGREGATE_SQL; - condition.setPrecision(Precision.SECONDS); - } - } else { - switch (condition.getPrecision()) { - case HOURS: - metricsAggregateTable = METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME; - queryStmt = GET_CLUSTER_AGGREGATE_HOURLY_SQL; - break; - default: - metricsAggregateTable = METRICS_CLUSTER_AGGREGATE_TABLE_NAME; - queryStmt = GET_CLUSTER_AGGREGATE_SQL; - } - } - - StringBuilder sb = new StringBuilder(queryStmt); + StringBuilder sb = new StringBuilder(GET_CLUSTER_AGGREGATE_SQL); sb.append(" WHERE "); sb.append(condition.getConditionClause()); sb.append(" ORDER BY METRIC_NAME, SERVER_TIME"); @@ -459,7 +388,7 @@ public class PhoenixTransactSQL { String query = String.format(sb.toString(), PhoenixTransactSQL.getNaiveTimeRangeHint(condition.getStartTime(), - NATIVE_TIME_RANGE_DELTA), metricsAggregateTable); + NATIVE_TIME_RANGE_DELTA)); if (LOG.isDebugEnabled()) { LOG.debug("SQL => " + query + ", condition => " + condition); } @@ -553,8 +482,6 @@ public class PhoenixTransactSQL { boolean isGrouped(); void setStatement(String statement); String getHostname(); - Precision getPrecision(); - void setPrecision(Precision precision); String getAppId(); String getInstanceId(); String getConditionClause(); @@ -576,7 +503,6 @@ public class PhoenixTransactSQL { String instanceId; Long startTime; Long endTime; - Precision precision; Integer limit; boolean grouped; boolean noLimit = false; @@ -585,15 +511,14 @@ public class PhoenixTransactSQL { Set<String> orderByColumns = new LinkedHashSet<String>(); DefaultCondition(List<String> metricNames, String hostname, String appId, - String instanceId, Long startTime, Long endTime, Precision precision, - Integer limit, boolean grouped) { + String instanceId, Long startTime, Long endTime, Integer limit, + boolean grouped) { this.metricNames = metricNames; this.hostname = hostname; this.appId = appId; this.instanceId = instanceId; this.startTime = startTime; this.endTime = endTime; - this.precision = precision; this.limit = limit; this.grouped = grouped; } @@ -666,14 +591,6 @@ public class PhoenixTransactSQL { public String getHostname() { return hostname == null || hostname.isEmpty() ? null : hostname; } - - public Precision getPrecision() { - return precision; - } - - public void setPrecision(Precision precision) { - this.precision = precision; - } public String getAppId() { if (appId != null && !appId.isEmpty()) { @@ -791,10 +708,9 @@ public class PhoenixTransactSQL { LikeCondition(List<String> metricNames, String hostname, String appId, String instanceId, Long startTime, - Long endTime, Precision precision, Integer limit, - boolean grouped) { + Long endTime, Integer limit, boolean grouped) { super(metricNames, hostname, appId, instanceId, startTime, endTime, - precision, limit, grouped); + limit, grouped); } @Override @@ -864,16 +780,6 @@ public class PhoenixTransactSQL { } @Override - public Precision getPrecision() { - return adaptee.getPrecision(); - } - - @Override - public void setPrecision(Precision precision) { - adaptee.setPrecision(precision); - } - - @Override public String getAppId() { return adaptee.getAppId(); } http://git-wip-us.apache.org/repos/asf/ambari/blob/c1402587/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/Precision.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/Precision.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/Precision.java deleted file mode 100644 index 8fd0849..0000000 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/Precision.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline; - -/** - * Is used to determine metrics aggregate table. - * - * @see org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.TimelineWebServices#getTimelineMetric - * @see org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.TimelineWebServices#getTimelineMetrics - */ -public enum Precision { - SECONDS, - MINUTES, - HOURS; - - public static Precision getPrecision(String precision) throws IllegalArgumentException { - if (precision == null ) { - return null; - } - return Precision.valueOf(precision.toUpperCase()); - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/c1402587/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregator.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregator.java index 459e612..f4f895f 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregator.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregator.java @@ -80,7 +80,7 @@ public class TimelineMetricAggregator extends AbstractTimelineAggregator { @Override protected Condition prepareMetricQueryCondition(long startTime, long endTime) { Condition condition = new DefaultCondition(null, null, null, null, startTime, - endTime, null, null, true); + endTime, null, true); condition.setNoLimit(); condition.setFetchSize(resultsetFetchSize); condition.setStatement(String.format(GET_METRIC_AGGREGATE_ONLY_SQL, http://git-wip-us.apache.org/repos/asf/ambari/blob/c1402587/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregator.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregator.java index c6bd8fc..e291f36 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregator.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregator.java @@ -93,7 +93,7 @@ public class TimelineMetricClusterAggregator extends AbstractTimelineAggregator @Override protected Condition prepareMetricQueryCondition(long startTime, long endTime) { Condition condition = new DefaultCondition(null, null, null, null, startTime, - endTime, null, null, true); + endTime, null, true); condition.setNoLimit(); condition.setFetchSize(resultsetFetchSize); condition.setStatement(String.format(GET_METRIC_SQL, http://git-wip-us.apache.org/repos/asf/ambari/blob/c1402587/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregatorHourly.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregatorHourly.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregatorHourly.java index 350404f..1d5c5a4 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregatorHourly.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregatorHourly.java @@ -34,7 +34,6 @@ import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.ti import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.DefaultCondition; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.GET_CLUSTER_AGGREGATE_SQL; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_TABLE_NAME; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_HOUR_CHECKPOINT_CUTOFF_INTERVAL; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_HOUR_CHECKPOINT_CUTOFF_MULTIPLIER; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_HOUR_DISABLED; @@ -92,12 +91,11 @@ public class TimelineMetricClusterAggregatorHourly extends protected Condition prepareMetricQueryCondition(long startTime, long endTime) { Condition condition = new DefaultCondition(null, null, null, null, startTime, - endTime, null, null, true); + endTime, null, true); condition.setNoLimit(); condition.setFetchSize(resultsetFetchSize); condition.setStatement(String.format(GET_CLUSTER_AGGREGATE_SQL, - PhoenixTransactSQL.getNaiveTimeRangeHint(startTime, NATIVE_TIME_RANGE_DELTA), - METRICS_CLUSTER_AGGREGATE_TABLE_NAME)); + PhoenixTransactSQL.getNaiveTimeRangeHint(startTime, NATIVE_TIME_RANGE_DELTA))); condition.addOrderByColumn("METRIC_NAME"); condition.addOrderByColumn("APP_ID"); condition.addOrderByColumn("INSTANCE_ID"); http://git-wip-us.apache.org/repos/asf/ambari/blob/c1402587/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java index fa52797..5224450 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java @@ -34,7 +34,6 @@ public interface TimelineMetricStore { * @param instanceId Application instance id. * @param startTime Start timestamp * @param endTime End timestamp - * @param precision Precision [ seconds, minutes, hours ] * @param limit Override default result limit * @param groupedByHosts Group {@link TimelineMetric} by metric name, hostname, * app id and instance id @@ -44,7 +43,7 @@ public interface TimelineMetricStore { */ TimelineMetrics getTimelineMetrics(List<String> metricNames, String hostname, String applicationId, String instanceId, Long startTime, - Long endTime, Precision precision, Integer limit, boolean groupedByHosts) + Long endTime, Integer limit, boolean groupedByHosts) throws SQLException, IOException; @@ -54,7 +53,7 @@ public interface TimelineMetricStore { */ TimelineMetric getTimelineMetric(String metricName, String hostname, String applicationId, String instanceId, Long startTime, - Long endTime, Precision precision, Integer limit) + Long endTime, Integer limit) throws SQLException, IOException; http://git-wip-us.apache.org/repos/asf/ambari/blob/c1402587/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java index 351db62..c9d56fc 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java @@ -30,7 +30,6 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents; import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; -import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.Precision; import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.EntityIdentifier; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper; @@ -296,20 +295,17 @@ public class TimelineWebServices { @QueryParam("hostname") String hostname, @QueryParam("startTime") String startTime, @QueryParam("endTime") String endTime, - @QueryParam("precision") String precision, @QueryParam("limit") String limit ) { init(res); try { return timelineMetricStore.getTimelineMetric(metricName, hostname, appId, instanceId, parseLongStr(startTime), parseLongStr(endTime), - Precision.getPrecision(precision), parseIntStr(limit)); + parseIntStr(limit)); + } catch (NumberFormatException ne) { throw new BadRequestException("startTime and limit should be numeric " + "values"); - } catch (IllegalArgumentException iae) { - throw new BadRequestException("precision should be seconds, minutes " + - "or hours"); } catch (SQLException sql) { throw new WebApplicationException(sql, Response.Status.INTERNAL_SERVER_ERROR); @@ -329,7 +325,6 @@ public class TimelineWebServices { * @param instanceId Application instance id. * @param hostname Hostname where the metrics originated. * @param startTime Start time for the metric records retrieved. - * @param precision Precision [ seconds, minutes, hours ] * @param limit limit on total number of {@link TimelineMetric} records * retrieved. * @return {@link TimelineMetrics} @@ -346,7 +341,6 @@ public class TimelineWebServices { @QueryParam("hostname") String hostname, @QueryParam("startTime") String startTime, @QueryParam("endTime") String endTime, - @QueryParam("precision") String precision, @QueryParam("limit") String limit, @QueryParam("grouped") String grouped ) { @@ -355,21 +349,16 @@ public class TimelineWebServices { LOG.debug("Request for metrics => metricNames: " + metricNames + ", " + "appId: " + appId + ", instanceId: " + instanceId + ", " + "hostname: " + hostname + ", startTime: " + startTime + ", " + - "endTime: " + endTime + ", " + - "precision: " + precision); + "endTime: " + endTime); return timelineMetricStore.getTimelineMetrics( parseListStr(metricNames, ","), hostname, appId, instanceId, - parseLongStr(startTime), parseLongStr(endTime), - Precision.getPrecision(precision), parseIntStr(limit), + parseLongStr(startTime), parseLongStr(endTime), parseIntStr(limit), parseBoolean(grouped)); } catch (NumberFormatException ne) { throw new BadRequestException("startTime and limit should be numeric " + "values"); - } catch (IllegalArgumentException iae) { - throw new BadRequestException("precision should be seconds, minutes " + - "or hours"); } catch (SQLException sql) { throw new WebApplicationException(sql, Response.Status.INTERNAL_SERVER_ERROR); http://git-wip-us.apache.org/repos/asf/ambari/blob/c1402587/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractMiniHBaseClusterTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractMiniHBaseClusterTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractMiniHBaseClusterTest.java index b11a977..2ca7669 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractMiniHBaseClusterTest.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractMiniHBaseClusterTest.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline; -import org.apache.hadoop.conf.Configuration; +import com.google.common.collect.Maps; import org.apache.hadoop.hbase.HConstants; import org.apache.phoenix.hbase.index.write.IndexWriterUtils; import org.apache.phoenix.query.BaseTest; @@ -36,8 +36,6 @@ import java.sql.Statement; import java.util.HashMap; import java.util.Map; import java.util.Properties; - -import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.LOG; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.assertj.core.api.Assertions.assertThat; @@ -112,26 +110,4 @@ public abstract class AbstractMiniHBaseClusterTest extends BaseTest { conn.close(); } - protected PhoenixHBaseAccessor createTestableHBaseAccessor() { - Configuration metricsConf = new Configuration(); - metricsConf.set( - TimelineMetricConfiguration.HBASE_COMPRESSION_SCHEME, "NONE"); - - return - new PhoenixHBaseAccessor( - new Configuration(), - metricsConf, - new ConnectionProvider() { - @Override - public Connection getConnection() { - Connection connection = null; - try { - connection = DriverManager.getConnection(getUrl()); - } catch (SQLException e) { - LOG.warn("Unable to connect to HBase store using Phoenix.", e); - } - return connection; - } - }); - } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/c1402587/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITClusterAggregator.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITClusterAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITClusterAggregator.java index 2164d01..2da9c82 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITClusterAggregator.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITClusterAggregator.java @@ -19,15 +19,19 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; +import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics; import org.junit.After; import org.junit.Before; import org.junit.Test; import java.sql.Connection; +import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -36,10 +40,8 @@ import static junit.framework.Assert.fail; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.Condition; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.DefaultCondition; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.GET_CLUSTER_AGGREGATE_SQL; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_TABLE_NAME; +import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.LOG; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.NATIVE_TIME_RANGE_DELTA; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.MetricTestHelper.prepareSingleTimelineMetric; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.MetricTestHelper.createEmptyTimelineClusterMetric; public class ITClusterAggregator extends AbstractMiniHBaseClusterTest { private Connection conn; @@ -95,10 +97,9 @@ public class ITClusterAggregator extends AbstractMiniHBaseClusterTest { //THEN Condition condition = new DefaultCondition(null, null, null, null, startTime, - endTime, null, null, true); + endTime, null, true); condition.setStatement(String.format(GET_CLUSTER_AGGREGATE_SQL, - PhoenixTransactSQL.getNaiveTimeRangeHint(startTime, NATIVE_TIME_RANGE_DELTA), - METRICS_CLUSTER_AGGREGATE_TABLE_NAME)); + PhoenixTransactSQL.getNaiveTimeRangeHint(startTime, NATIVE_TIME_RANGE_DELTA))); PreparedStatement pstmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt (conn, condition); @@ -156,10 +157,9 @@ public class ITClusterAggregator extends AbstractMiniHBaseClusterTest { //THEN Condition condition = new DefaultCondition(null, null, null, null, startTime, - endTime, null, null, true); + endTime, null, true); condition.setStatement(String.format(GET_CLUSTER_AGGREGATE_SQL, - PhoenixTransactSQL.getNaiveTimeRangeHint(startTime, NATIVE_TIME_RANGE_DELTA), - METRICS_CLUSTER_AGGREGATE_TABLE_NAME)); + PhoenixTransactSQL.getNaiveTimeRangeHint(startTime, NATIVE_TIME_RANGE_DELTA))); PreparedStatement pstmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt (conn, condition); @@ -205,13 +205,13 @@ public class ITClusterAggregator extends AbstractMiniHBaseClusterTest { Map<TimelineClusterMetric, MetricClusterAggregate> records = new HashMap<TimelineClusterMetric, MetricClusterAggregate>(); - records.put(createEmptyTimelineClusterMetric(ctime), + records.put(createEmptyTimelineMetric(ctime), new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0)); - records.put(createEmptyTimelineClusterMetric(ctime += minute), + records.put(createEmptyTimelineMetric(ctime += minute), new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0)); - records.put(createEmptyTimelineClusterMetric(ctime += minute), + records.put(createEmptyTimelineMetric(ctime += minute), new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0)); - records.put(createEmptyTimelineClusterMetric(ctime += minute), + records.put(createEmptyTimelineMetric(ctime += minute), new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0)); hdb.saveClusterAggregateRecords(records); @@ -249,24 +249,24 @@ public class ITClusterAggregator extends AbstractMiniHBaseClusterTest { Map<TimelineClusterMetric, MetricClusterAggregate> records = new HashMap<TimelineClusterMetric, MetricClusterAggregate>(); - records.put(createEmptyTimelineClusterMetric("disk_used", ctime), + records.put(createEmptyTimelineMetric("disk_used", ctime), new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0)); - records.put(createEmptyTimelineClusterMetric("disk_free", ctime), + records.put(createEmptyTimelineMetric("disk_free", ctime), new MetricClusterAggregate(1.0, 2, 0.0, 1.0, 1.0)); - records.put(createEmptyTimelineClusterMetric("disk_used", ctime += minute), + records.put(createEmptyTimelineMetric("disk_used", ctime += minute), new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0)); - records.put(createEmptyTimelineClusterMetric("disk_free", ctime), + records.put(createEmptyTimelineMetric("disk_free", ctime), new MetricClusterAggregate(1.0, 2, 0.0, 1.0, 1.0)); - records.put(createEmptyTimelineClusterMetric("disk_used", ctime += minute), + records.put(createEmptyTimelineMetric("disk_used", ctime += minute), new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0)); - records.put(createEmptyTimelineClusterMetric("disk_free", ctime), + records.put(createEmptyTimelineMetric("disk_free", ctime), new MetricClusterAggregate(1.0, 2, 0.0, 1.0, 1.0)); - records.put(createEmptyTimelineClusterMetric("disk_used", ctime += minute), + records.put(createEmptyTimelineMetric("disk_used", ctime += minute), new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0)); - records.put(createEmptyTimelineClusterMetric("disk_free", ctime), + records.put(createEmptyTimelineMetric("disk_free", ctime), new MetricClusterAggregate(1.0, 2, 0.0, 1.0, 1.0)); hdb.saveClusterAggregateRecords(records); @@ -303,4 +303,83 @@ public class ITClusterAggregator extends AbstractMiniHBaseClusterTest { Statement stmt = conn.createStatement(); return stmt.executeQuery(query); } + + private TimelineClusterMetric createEmptyTimelineMetric(String name, + long startTime) { + TimelineClusterMetric metric = new TimelineClusterMetric(name, + "test_app", null, startTime, null); + + return metric; + } + + private TimelineClusterMetric createEmptyTimelineMetric(long startTime) { + return createEmptyTimelineMetric("disk_used", startTime); + } + + private MetricHostAggregate + createMetricHostAggregate(double max, double min, int numberOfSamples, + double sum) { + MetricHostAggregate expectedAggregate = + new MetricHostAggregate(); + expectedAggregate.setMax(max); + expectedAggregate.setMin(min); + expectedAggregate.setNumberOfSamples(numberOfSamples); + expectedAggregate.setSum(sum); + + return expectedAggregate; + } + + private PhoenixHBaseAccessor createTestableHBaseAccessor() { + Configuration metricsConf = new Configuration(); + metricsConf.set( + TimelineMetricConfiguration.HBASE_COMPRESSION_SCHEME, "NONE"); + + return + new PhoenixHBaseAccessor( + new Configuration(), + metricsConf, + new ConnectionProvider() { + @Override + public Connection getConnection() { + Connection connection = null; + try { + connection = DriverManager.getConnection(getUrl()); + } catch (SQLException e) { + LOG.warn("Unable to connect to HBase store using Phoenix.", e); + } + return connection; + } + }); + } + + private TimelineMetrics prepareSingleTimelineMetric(long startTime, + String host, + String metricName, + double val) { + TimelineMetrics m = new TimelineMetrics(); + m.setMetrics(Arrays.asList( + createTimelineMetric(startTime, metricName, host, val))); + + return m; + } + + private TimelineMetric createTimelineMetric(long startTime, + String metricName, + String host, + double val) { + TimelineMetric m = new TimelineMetric(); + m.setAppId("host"); + m.setHostName(host); + m.setMetricName(metricName); + m.setStartTime(startTime); + Map<Long, Double> vals = new HashMap<Long, Double>(); + vals.put(startTime + 15000l, val); + vals.put(startTime + 30000l, val); + vals.put(startTime + 45000l, val); + vals.put(startTime + 60000l, val); + + m.setMetricValues(vals); + + return m; + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/c1402587/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITMetricAggregator.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITMetricAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITMetricAggregator.java index 427e826..22e1363 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITMetricAggregator.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITMetricAggregator.java @@ -25,8 +25,10 @@ import org.junit.Before; import org.junit.Test; import java.sql.Connection; +import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; +import java.sql.SQLException; import java.sql.Statement; import java.util.Arrays; import java.util.Comparator; @@ -39,11 +41,11 @@ import static junit.framework.Assert.fail; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.Condition; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.DefaultCondition; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.GET_METRIC_AGGREGATE_ONLY_SQL; +import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.LOG; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.METRICS_AGGREGATE_HOURLY_TABLE_NAME; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.METRICS_AGGREGATE_MINUTE_TABLE_NAME; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.NATIVE_TIME_RANGE_DELTA; import static org.assertj.core.api.Assertions.assertThat; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.MetricTestHelper.createEmptyTimelineMetric; public class ITMetricAggregator extends AbstractMiniHBaseClusterTest { private Connection conn; @@ -84,7 +86,7 @@ public class ITMetricAggregator extends AbstractMiniHBaseClusterTest { hdb.insertMetricRecords(metricsSent); Condition queryCondition = new DefaultCondition(null, "local", null, null, - startTime, startTime + (15 * 60 * 1000), null, null, false); + startTime, startTime + (15 * 60 * 1000), null, false); TimelineMetrics recordRead = hdb.getMetricRecords(queryCondition); // THEN @@ -120,7 +122,7 @@ public class ITMetricAggregator extends AbstractMiniHBaseClusterTest { //THEN Condition condition = new DefaultCondition(null, null, null, null, startTime, - endTime, null, null, true); + endTime, null, true); condition.setStatement(String.format(GET_METRIC_AGGREGATE_ONLY_SQL, PhoenixTransactSQL.getNaiveTimeRangeHint(startTime, NATIVE_TIME_RANGE_DELTA), METRICS_AGGREGATE_MINUTE_TABLE_NAME)); @@ -129,7 +131,7 @@ public class ITMetricAggregator extends AbstractMiniHBaseClusterTest { (conn, condition); ResultSet rs = pstmt.executeQuery(); MetricHostAggregate expectedAggregate = - MetricTestHelper.createMetricHostAggregate(2.0, 0.0, 20, 15.0); + createMetricHostAggregate(2.0, 0.0, 20, 15.0); int count = 0; while (rs.next()) { @@ -170,7 +172,7 @@ public class ITMetricAggregator extends AbstractMiniHBaseClusterTest { long startTime = System.currentTimeMillis(); MetricHostAggregate expectedAggregate = - MetricTestHelper.createMetricHostAggregate(2.0, 0.0, 20, 15.0); + createMetricHostAggregate(2.0, 0.0, 20, 15.0); Map<TimelineMetric, MetricHostAggregate> aggMap = new HashMap<TimelineMetric, MetricHostAggregate>(); @@ -199,7 +201,7 @@ public class ITMetricAggregator extends AbstractMiniHBaseClusterTest { //THEN Condition condition = new DefaultCondition(null, null, null, null, startTime, - endTime, null, null, true); + endTime, null, true); condition.setStatement(String.format(GET_METRIC_AGGREGATE_ONLY_SQL, PhoenixTransactSQL.getNaiveTimeRangeHint(startTime, NATIVE_TIME_RANGE_DELTA), METRICS_AGGREGATE_HOURLY_TABLE_NAME)); @@ -224,6 +226,52 @@ public class ITMetricAggregator extends AbstractMiniHBaseClusterTest { } } + private TimelineMetric createEmptyTimelineMetric(long startTime) { + TimelineMetric metric = new TimelineMetric(); + metric.setMetricName("disk_used"); + metric.setAppId("test_app"); + metric.setHostName("test_host"); + metric.setTimestamp(startTime); + + return metric; + } + + private MetricHostAggregate + createMetricHostAggregate(double max, double min, int numberOfSamples, + double sum) { + MetricHostAggregate expectedAggregate = + new MetricHostAggregate(); + expectedAggregate.setMax(max); + expectedAggregate.setMin(min); + expectedAggregate.setNumberOfSamples(numberOfSamples); + expectedAggregate.setSum(sum); + + return expectedAggregate; + } + + private PhoenixHBaseAccessor createTestableHBaseAccessor() { + Configuration metricsConf = new Configuration(); + metricsConf.set( + TimelineMetricConfiguration.HBASE_COMPRESSION_SCHEME, "NONE"); + + return + new PhoenixHBaseAccessor( + new Configuration(), + metricsConf, + new ConnectionProvider() { + @Override + public Connection getConnection() { + Connection connection = null; + try { + connection = DriverManager.getConnection(getUrl()); + } catch (SQLException e) { + LOG.warn("Unable to connect to HBase store using Phoenix.", e); + } + return connection; + } + }); + } + private final static Comparator<TimelineMetric> TIME_IGNORING_COMPARATOR = new Comparator<TimelineMetric>() { @Override http://git-wip-us.apache.org/repos/asf/ambari/blob/c1402587/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java deleted file mode 100644 index cc3f63b..0000000 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java +++ /dev/null @@ -1,271 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; -import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; - -import static junit.framework.Assert.assertEquals; -import static junit.framework.Assert.assertTrue; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.METRICS_AGGREGATE_MINUTE_TABLE_NAME; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.MetricTestHelper.prepareSingleTimelineMetric; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.MetricTestHelper.createEmptyTimelineMetric; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.MetricTestHelper.createMetricHostAggregate; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.MetricTestHelper.createEmptyTimelineClusterMetric; - - -public class ITPhoenixHBaseAccessor extends AbstractMiniHBaseClusterTest { - private Connection conn; - private PhoenixHBaseAccessor hdb; - - @Before - public void setUp() throws Exception { - hdb = createTestableHBaseAccessor(); - // inits connection, starts mini cluster - conn = getConnection(getUrl()); - - hdb.initMetricSchema(); - } - - @After - public void tearDown() throws Exception { - Connection conn = getConnection(getUrl()); - Statement stmt = conn.createStatement(); - - stmt.execute("delete from METRIC_AGGREGATE"); - stmt.execute("delete from METRIC_AGGREGATE_HOURLY"); - stmt.execute("delete from METRIC_RECORD"); - stmt.execute("delete from METRIC_RECORD_HOURLY"); - stmt.execute("delete from METRIC_RECORD_MINUTE"); - conn.commit(); - - stmt.close(); - conn.close(); - } - - @Test - public void testGetMetricRecordsSeconds() throws IOException, SQLException { - // GIVEN - long startTime = System.currentTimeMillis(); - long ctime = startTime; - long minute = 60 * 1000; - hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local1", - "disk_free", 1)); - hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local2", - "disk_free", 2)); - ctime += minute; - hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local1", - "disk_free", 2)); - hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local2", - "disk_free", 1)); - - // WHEN - long endTime = ctime + minute; - PhoenixTransactSQL.Condition condition = new PhoenixTransactSQL.DefaultCondition( - Collections.singletonList("disk_free"), "local1", null, null, startTime, - endTime, Precision.SECONDS, null, true); - TimelineMetrics timelineMetrics = hdb.getMetricRecords(condition); - - //THEN - assertEquals(1, timelineMetrics.getMetrics().size()); - TimelineMetric metric = timelineMetrics.getMetrics().get(0); - - assertEquals("disk_free", metric.getMetricName()); - assertEquals("local1", metric.getHostName()); - assertEquals(8, metric.getMetricValues().size()); - } - - @Test - public void testGetMetricRecordsMinutes() throws IOException, SQLException { - // GIVEN - TimelineMetricAggregator aggregatorMinute = TimelineMetricAggregatorFactory - .createTimelineMetricAggregatorMinute(hdb, new Configuration()); - - long startTime = System.currentTimeMillis(); - long ctime = startTime; - long minute = 60 * 1000; - hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local1", - "disk_free", 1)); - hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime + minute, "local1", - "disk_free", 2)); - hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local2", - "disk_free", 2)); - long endTime = ctime + minute; - boolean success = aggregatorMinute.doWork(startTime, endTime); - assertTrue(success); - - // WHEN - PhoenixTransactSQL.Condition condition = new PhoenixTransactSQL.DefaultCondition( - Collections.singletonList("disk_free"), "local1", null, null, startTime, - endTime, Precision.MINUTES, null, false); - TimelineMetrics timelineMetrics = hdb.getMetricRecords(condition); - - //THEN - assertEquals(1, timelineMetrics.getMetrics().size()); - TimelineMetric metric = timelineMetrics.getMetrics().get(0); - - assertEquals("disk_free", metric.getMetricName()); - assertEquals("local1", metric.getHostName()); - assertEquals(1, metric.getMetricValues().size()); - Iterator<Map.Entry<Long, Double>> iterator = metric.getMetricValues().entrySet().iterator(); - assertEquals(1.5, iterator.next().getValue(), 0.00001); - } - - @Test - public void testGetMetricRecordsHours() throws IOException, SQLException { - // GIVEN - TimelineMetricAggregator aggregator = TimelineMetricAggregatorFactory - .createTimelineMetricAggregatorHourly(hdb, new Configuration()); - - MetricHostAggregate expectedAggregate = - createMetricHostAggregate(2.0, 0.0, 20, 15.0); - Map<TimelineMetric, MetricHostAggregate> - aggMap = new HashMap<TimelineMetric, - MetricHostAggregate>(); - - long startTime = System.currentTimeMillis(); - int min_5 = 5 * 60 * 1000; - long ctime = startTime - min_5; - aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate); - aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate); - aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate); - aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate); - aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate); - aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate); - aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate); - aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate); - aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate); - aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate); - aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate); - aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate); - - hdb.saveHostAggregateRecords(aggMap, METRICS_AGGREGATE_MINUTE_TABLE_NAME); - long endTime = ctime + min_5; - boolean success = aggregator.doWork(startTime, endTime); - assertTrue(success); - - // WHEN - PhoenixTransactSQL.Condition condition = new PhoenixTransactSQL.DefaultCondition( - Collections.singletonList("disk_used"), "test_host", "test_app", null, - startTime, endTime, Precision.HOURS, null, true); - TimelineMetrics timelineMetrics = hdb.getMetricRecords(condition); - - //THEN - assertEquals(1, timelineMetrics.getMetrics().size()); - TimelineMetric metric = timelineMetrics.getMetrics().get(0); - - assertEquals("disk_used", metric.getMetricName()); - assertEquals("test_host", metric.getHostName()); - assertEquals(1, metric.getMetricValues().size()); - Iterator<Map.Entry<Long, Double>> iterator = metric.getMetricValues().entrySet().iterator(); - assertEquals(0.75, iterator.next().getValue(), 0.00001); - } - - @Test - public void testGetClusterMetricRecordsSeconds() throws Exception { - // GIVEN - TimelineMetricClusterAggregator agg = - new TimelineMetricClusterAggregator(hdb, new Configuration()); - - long startTime = System.currentTimeMillis(); - long ctime = startTime + 1; - long minute = 60 * 1000; - hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local1", - "disk_free", 1)); - hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local2", - "disk_free", 2)); - ctime += minute; - hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local1", - "disk_free", 2)); - hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local2", - "disk_free", 1)); - - long endTime = ctime + minute + 1; - boolean success = agg.doWork(startTime, endTime); - assertTrue(success); - - // WHEN - PhoenixTransactSQL.Condition condition = new PhoenixTransactSQL.DefaultCondition( - Collections.singletonList("disk_free"), null, null, null, - startTime, endTime, Precision.SECONDS, null, true); - TimelineMetrics timelineMetrics = hdb.getAggregateMetricRecords(condition); - - //THEN - assertEquals(1, timelineMetrics.getMetrics().size()); - TimelineMetric metric = timelineMetrics.getMetrics().get(0); - - assertEquals("disk_free", metric.getMetricName()); - assertEquals(8, metric.getMetricValues().size()); - assertEquals(1.5, metric.getMetricValues().values().iterator().next(), 0.00001); - } - - @Test - public void testGetClusterMetricRecordsHours() throws Exception { - // GIVEN - TimelineMetricClusterAggregatorHourly agg = - new TimelineMetricClusterAggregatorHourly(hdb, new Configuration()); - - long startTime = System.currentTimeMillis(); - long ctime = startTime; - long minute = 60 * 1000; - - Map<TimelineClusterMetric, MetricClusterAggregate> records = - new HashMap<TimelineClusterMetric, MetricClusterAggregate>(); - - records.put(createEmptyTimelineClusterMetric(ctime), - new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0)); - records.put(createEmptyTimelineClusterMetric(ctime += minute), - new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0)); - records.put(createEmptyTimelineClusterMetric(ctime += minute), - new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0)); - records.put(createEmptyTimelineClusterMetric(ctime += minute), - new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0)); - - hdb.saveClusterAggregateRecords(records); - boolean success = agg.doWork(startTime, ctime + minute); - assertTrue(success); - - // WHEN - PhoenixTransactSQL.Condition condition = new PhoenixTransactSQL.DefaultCondition( - Collections.singletonList("disk_used"), null, null, null, - startTime, ctime + minute, Precision.HOURS, null, true); - TimelineMetrics timelineMetrics = hdb.getAggregateMetricRecords(condition); - - // THEN - assertEquals(1, timelineMetrics.getMetrics().size()); - TimelineMetric metric = timelineMetrics.getMetrics().get(0); - - assertEquals("disk_used", metric.getMetricName()); - assertEquals("test_app", metric.getAppId()); - assertEquals(1, metric.getMetricValues().size()); - assertEquals(2.0, metric.getMetricValues().values().iterator().next(), 0.00001); - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/c1402587/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java deleted file mode 100644 index 440a6b6..0000000 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java +++ /dev/null @@ -1,96 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline; - -import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; -import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; - -public class MetricTestHelper { - - public static MetricHostAggregate - createMetricHostAggregate(double max, double min, int numberOfSamples, - double sum) { - MetricHostAggregate expectedAggregate = - new MetricHostAggregate(); - expectedAggregate.setMax(max); - expectedAggregate.setMin(min); - expectedAggregate.setNumberOfSamples(numberOfSamples); - expectedAggregate.setSum(sum); - - return expectedAggregate; - } - - public static TimelineMetrics prepareSingleTimelineMetric(long startTime, - String host, - String metricName, - double val) { - TimelineMetrics m = new TimelineMetrics(); - m.setMetrics(Arrays.asList( - createTimelineMetric(startTime, metricName, host, val))); - - return m; - } - - - public static TimelineMetric createTimelineMetric(long startTime, - String metricName, - String host, - double val) { - TimelineMetric m = new TimelineMetric(); - m.setAppId("host"); - m.setHostName(host); - m.setMetricName(metricName); - m.setStartTime(startTime); - Map<Long, Double> vals = new HashMap<Long, Double>(); - vals.put(startTime + 15000l, val); - vals.put(startTime + 30000l, val); - vals.put(startTime + 45000l, val); - vals.put(startTime + 60000l, val); - - m.setMetricValues(vals); - - return m; - } - - public static TimelineMetric createEmptyTimelineMetric(long startTime) { - TimelineMetric metric = new TimelineMetric(); - metric.setMetricName("disk_used"); - metric.setAppId("test_app"); - metric.setHostName("test_host"); - metric.setTimestamp(startTime); - - return metric; - } - - public static TimelineClusterMetric createEmptyTimelineClusterMetric( - String name, long startTime) { - TimelineClusterMetric metric = new TimelineClusterMetric(name, - "test_app", null, startTime, null); - - return metric; - } - - public static TimelineClusterMetric createEmptyTimelineClusterMetric( - long startTime) { - return createEmptyTimelineClusterMetric("disk_used", startTime); - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/c1402587/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestClusterSuite.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestClusterSuite.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestClusterSuite.java index 640604d..0722ccd 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestClusterSuite.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestClusterSuite.java @@ -24,7 +24,7 @@ import org.junit.runners.Suite; import static org.junit.runners.Suite.SuiteClasses; @RunWith(Suite.class) -@SuiteClasses({ITMetricAggregator.class, ITClusterAggregator.class, ITPhoenixHBaseAccessor.class}) +@SuiteClasses({ITMetricAggregator.class, ITClusterAggregator.class}) public class TestClusterSuite { } http://git-wip-us.apache.org/repos/asf/ambari/blob/c1402587/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestPhoenixTransactSQL.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestPhoenixTransactSQL.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestPhoenixTransactSQL.java index af472c7..333b13b 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestPhoenixTransactSQL.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestPhoenixTransactSQL.java @@ -17,13 +17,9 @@ */ package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline; -import org.easymock.Capture; import org.junit.Assert; import org.junit.Test; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.SQLException; import java.util.Arrays; import java.util.Collections; @@ -32,19 +28,12 @@ import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.ti import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.LikeCondition; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.SplitByMetricNamesCondition; -import static org.easymock.EasyMock.createNiceMock; -import static org.easymock.EasyMock.expect; -import static org.easymock.EasyMock.replay; -import static org.easymock.EasyMock.verify; - -import org.easymock.EasyMock; - public class TestPhoenixTransactSQL { @Test public void testConditionClause() throws Exception { Condition condition = new DefaultCondition( Arrays.asList("cpu_user", "mem_free"), "h1", "a1", "i1", - 1407959718L, 1407959918L, null, null, false); + 1407959718L, 1407959918L, null, false); String preparedClause = condition.getConditionClause(); String expectedClause = "METRIC_NAME IN (?, ?) AND HOSTNAME = ? AND " + @@ -58,7 +47,7 @@ public class TestPhoenixTransactSQL { public void testSplitByMetricNamesCondition() throws Exception { Condition c = new DefaultCondition( Arrays.asList("cpu_user", "mem_free"), "h1", "a1", "i1", - 1407959718L, 1407959918L, null, null, false); + 1407959718L, 1407959918L, null, false); SplitByMetricNamesCondition condition = new SplitByMetricNamesCondition(c); condition.setCurrentMetric(c.getMetricNames().get(0)); @@ -75,7 +64,7 @@ public class TestPhoenixTransactSQL { public void testLikeConditionClause() throws Exception { Condition condition = new LikeCondition( Arrays.asList("cpu_user", "mem_free"), "h1", "a1", "i1", - 1407959718L, 1407959918L, null, null, false); + 1407959718L, 1407959918L, null, false); String preparedClause = condition.getConditionClause(); String expectedClause = "(METRIC_NAME LIKE ? OR METRIC_NAME LIKE ?) AND HOSTNAME = ? AND " + @@ -87,7 +76,7 @@ public class TestPhoenixTransactSQL { condition = new LikeCondition( Collections.<String>emptyList(), "h1", "a1", "i1", - 1407959718L, 1407959918L, null, null, false); + 1407959718L, 1407959918L, null, false); preparedClause = condition.getConditionClause(); expectedClause = " HOSTNAME = ? AND " + @@ -99,7 +88,7 @@ public class TestPhoenixTransactSQL { condition = new LikeCondition( null, "h1", "a1", "i1", - 1407959718L, 1407959918L, null, null, false); + 1407959718L, 1407959918L, null, false); preparedClause = condition.getConditionClause(); expectedClause = " HOSTNAME = ? AND " + @@ -111,7 +100,7 @@ public class TestPhoenixTransactSQL { condition = new LikeCondition( Arrays.asList("cpu_user"), "h1", "a1", "i1", - 1407959718L, 1407959918L, null, null, false); + 1407959718L, 1407959918L, null, false); preparedClause = condition.getConditionClause(); expectedClause = "(METRIC_NAME LIKE ?) AND HOSTNAME = ? AND " + @@ -123,7 +112,7 @@ public class TestPhoenixTransactSQL { condition = new LikeCondition( Arrays.asList("cpu_user", "mem_free", "cpu_aidle"), "h1", "a1", "i1", - 1407959718L, 1407959918L, null, null, false); + 1407959718L, 1407959918L, null, false); preparedClause = condition.getConditionClause(); expectedClause = "(METRIC_NAME LIKE ? OR METRIC_NAME LIKE ? OR METRIC_NAME LIKE ?) AND HOSTNAME = ? AND " + @@ -132,112 +121,4 @@ public class TestPhoenixTransactSQL { Assert.assertNotNull(preparedClause); Assert.assertEquals(expectedClause, preparedClause); } - - @Test - public void testPrepareGetAggregatePrecisionMINUTES() throws SQLException { - Condition condition = new DefaultCondition( - Arrays.asList("cpu_user", "mem_free"), "h1", "a1", "i1", - 1407959718L, 1407959918L, Precision.MINUTES, null, false); - Connection connection = createNiceMock(Connection.class); - PreparedStatement preparedStatement = createNiceMock(PreparedStatement.class); - Capture<String> stmtCapture = new Capture<String>(); - expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture)))) - .andReturn(preparedStatement); - - replay(connection, preparedStatement); - PhoenixTransactSQL.prepareGetAggregateSqlStmt(connection, condition); - String stmt = stmtCapture.getValue(); - Assert.assertTrue(stmt.contains("FROM METRIC_AGGREGATE")); - verify(connection, preparedStatement); - } - - @Test - public void testPrepareGetAggregateNoPrecision() throws SQLException { - Condition condition = new DefaultCondition( - Arrays.asList("cpu_user", "mem_free"), "h1", "a1", "i1", - 1407959718L, 1407959918L, null, null, false); - Connection connection = createNiceMock(Connection.class); - PreparedStatement preparedStatement = createNiceMock(PreparedStatement.class); - Capture<String> stmtCapture = new Capture<String>(); - expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture)))) - .andReturn(preparedStatement); - - replay(connection, preparedStatement); - PhoenixTransactSQL.prepareGetAggregateSqlStmt(connection, condition); - String stmt = stmtCapture.getValue(); - Assert.assertTrue(stmt.contains("FROM METRIC_AGGREGATE")); - verify(connection, preparedStatement); - } - - @Test - public void testPrepareGetAggregatePrecisionHours() throws SQLException { - Condition condition = new DefaultCondition( - Arrays.asList("cpu_user", "mem_free"), "h1", "a1", "i1", - 1407959718L, 1407959918L, Precision.HOURS, null, false); - Connection connection = createNiceMock(Connection.class); - PreparedStatement preparedStatement = createNiceMock(PreparedStatement.class); - Capture<String> stmtCapture = new Capture<String>(); - expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture)))) - .andReturn(preparedStatement); - - replay(connection, preparedStatement); - PhoenixTransactSQL.prepareGetAggregateSqlStmt(connection, condition); - String stmt = stmtCapture.getValue(); - Assert.assertTrue(stmt.contains("FROM METRIC_AGGREGATE_HOURLY")); - verify(connection, preparedStatement); - } - - @Test - public void testPrepareGetMetricsPrecisionMinutes() throws SQLException { - Condition condition = new DefaultCondition( - Arrays.asList("cpu_user", "mem_free"), "h1", "a1", "i1", - 1407959718L, 1407959918L, Precision.MINUTES, null, false); - Connection connection = createNiceMock(Connection.class); - PreparedStatement preparedStatement = createNiceMock(PreparedStatement.class); - Capture<String> stmtCapture = new Capture<String>(); - expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture)))) - .andReturn(preparedStatement); - - replay(connection, preparedStatement); - PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition); - String stmt = stmtCapture.getValue(); - Assert.assertTrue(stmt.contains("FROM METRIC_RECORD_MINUTE")); - verify(connection, preparedStatement); - } - - @Test - public void testPrepareGetMetricsNoPrecision() throws SQLException { - Condition condition = new DefaultCondition( - Arrays.asList("cpu_user", "mem_free"), "h1", "a1", "i1", - 1407959718L, 1407959918L, null, null, false); - Connection connection = createNiceMock(Connection.class); - PreparedStatement preparedStatement = createNiceMock(PreparedStatement.class); - Capture<String> stmtCapture = new Capture<String>(); - expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture)))) - .andReturn(preparedStatement); - - replay(connection, preparedStatement); - PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition); - String stmt = stmtCapture.getValue(); - Assert.assertTrue(stmt.contains("FROM METRIC_RECORD")); - verify(connection, preparedStatement); - } - - @Test - public void testPrepareGetMetricsPrecisionHours() throws SQLException { - Condition condition = new DefaultCondition( - Arrays.asList("cpu_user", "mem_free"), "h1", "a1", "i1", - 1407959718L, 1407959918L, Precision.HOURS, null, false); - Connection connection = createNiceMock(Connection.class); - PreparedStatement preparedStatement = createNiceMock(PreparedStatement.class); - Capture<String> stmtCapture = new Capture<String>(); - expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture)))) - .andReturn(preparedStatement); - - replay(connection, preparedStatement); - PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition); - String stmt = stmtCapture.getValue(); - Assert.assertTrue(stmt.contains("FROM METRIC_RECORD_HOURLY")); - verify(connection, preparedStatement); - } } http://git-wip-us.apache.org/repos/asf/ambari/blob/c1402587/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java index a2d55af..c893314 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java @@ -30,7 +30,7 @@ public class TestTimelineMetricStore implements TimelineMetricStore { @Override public TimelineMetrics getTimelineMetrics(List<String> metricNames, String hostname, String applicationId, String instanceId, Long startTime, - Long endTime, Precision precision, Integer limit, boolean groupedByHost) throws SQLException, + Long endTime, Integer limit, boolean groupedByHost) throws SQLException, IOException { TimelineMetrics timelineMetrics = new TimelineMetrics(); List<TimelineMetric> metricList = new ArrayList<TimelineMetric>(); @@ -67,7 +67,7 @@ public class TestTimelineMetricStore implements TimelineMetricStore { @Override public TimelineMetric getTimelineMetric(String metricName, String hostname, String applicationId, String instanceId, Long startTime, Long endTime, - Precision precision, Integer limit) throws SQLException, IOException { + Integer limit) throws SQLException, IOException { return null; } http://git-wip-us.apache.org/repos/asf/ambari/blob/c1402587/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java index b093a2a..2b93190 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java @@ -382,7 +382,7 @@ public class TestTimelineWebServices extends JerseyTest { public void testGetMetrics() throws Exception { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("timeline") - .path("metrics").queryParam("metricNames", "cpu_user").queryParam("precision", "seconds") + .path("metrics").queryParam("metricNames", "cpu_user") .accept(MediaType.APPLICATION_JSON) .get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); http://git-wip-us.apache.org/repos/asf/ambari/blob/c1402587/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java index a1d44f2..bf68986 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java @@ -95,8 +95,6 @@ public abstract class AbstractProviderModule implements ProviderModule, private static final Map<Service.Type, Map<String, String[]>> serviceDesiredProperties = new EnumMap<Service.Type, Map<String, String[]>>(Service.Type.class); private static final Map<String, Service.Type> componentServiceMap = new HashMap<String, Service.Type>(); - private static final String COLLECTOR_DEFAULT_PORT = "6188"; - private static final Map<String, Map<String, String[]>> jmxDesiredProperties = new HashMap<String, Map<String, String[]>>(); private volatile Map<String, String> clusterCoreSiteConfigVersionMap = new HashMap<String, String>(); private volatile Map<String, String> clusterJmxProtocolMap = new HashMap<String, String>(); @@ -353,7 +351,7 @@ public abstract class AbstractProviderModule implements ProviderModule, if (!configProperties.isEmpty()) { clusterMetricServerPort = getPortString(configProperties.get("METRIC_COLLECTOR")); } else { - clusterMetricServerPort = COLLECTOR_DEFAULT_PORT; + clusterMetricServerPort = "8188"; } } http://git-wip-us.apache.org/repos/asf/ambari/blob/c1402587/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java index 8d29b4e..e9aac45 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java @@ -55,7 +55,6 @@ public abstract class AMSPropertyProvider extends MetricsPropertyProvider { private static ObjectMapper mapper; private final static ObjectReader timelineObjectReader; private static final String METRIC_REGEXP_PATTERN = "\\([^)]*\\)"; - private static final int COLLECTOR_DEFAULT_PORT = 6188; static { TIMELINE_APPID_MAP.put("HBASE_MASTER", "HBASE"); @@ -391,7 +390,7 @@ public abstract class AMSPropertyProvider extends MetricsPropertyProvider { if (metricsRequest == null) { metricsRequest = new MetricsRequest(temporalInfo, getAMSUriBuilder(collectorHostName, - collectorPort != null ? Integer.parseInt(collectorPort) : COLLECTOR_DEFAULT_PORT)); + collectorPort != null ? Integer.parseInt(collectorPort) : 8188)); requests.put(temporalInfo, metricsRequest); } metricsRequest.putResource(getHostName(resource), resource);
