This is an automated email from the ASF dual-hosted git repository.

avijayan pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git


The following commit(s) were added to refs/heads/trunk by this push:
     new a044354  AMBARI-23567 : Add ability to skip in memory cluster 
aggregation on certain metrics.
a044354 is described below

commit a044354cb7a0523b3d7e5d082411425b4bbbe70f
Author: Aravindan Vijayan <avija...@hortonworks.com>
AuthorDate: Thu Apr 12 14:25:02 2018 -0700

    AMBARI-23567 : Add ability to skip in memory cluster aggregation on certain 
metrics.
---
 .../metrics/timeline/PhoenixHBaseAccessor.java            | 15 +--------------
 .../metrics/timeline/TimelineMetricsIgniteCache.java      | 11 ++++++++++-
 .../metrics/timeline/aggregators/AggregatorUtils.java     | 14 ++++++++++++++
 .../TimelineMetricClusterAggregatorSecond.java            |  6 +++---
 .../AMBARI_METRICS/0.1.0/configuration/ams-site.xml       |  2 +-
 5 files changed, 29 insertions(+), 19 deletions(-)

diff --git 
a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
 
b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
index 69307d1..4a0c605 100644
--- 
a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
+++ 
b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
@@ -47,6 +47,7 @@ import static 
org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.ti
 import static 
org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_PRECISION_TABLE_DURABILITY;
 import static 
org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_PRECISION_TABLE_HBASE_BLOCKING_STORE_FILES;
 import static 
org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRIC_AGGREGATOR_SINK_CLASS;
+import static 
org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils.getJavaRegexFromSqlRegex;
 import static 
org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.ALTER_METRICS_METADATA_TABLE;
 import static 
org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.CONTAINER_METRICS_TABLE_NAME;
 import static 
org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.CREATE_CONTAINER_METRICS_TABLE_SQL;
@@ -1275,20 +1276,6 @@ public class PhoenixHBaseAccessor {
     return null;
   }
 
-  public String getJavaRegexFromSqlRegex(String sqlRegex) {
-    String javaRegEx;
-    if (sqlRegex.contains("*") || sqlRegex.contains("__%")) {
-      //Special case handling for metric name with * and __%.
-      //For example, dfs.NNTopUserOpCounts.windowMs=300000.op=*.user=%.count
-      // or dfs.NNTopUserOpCounts.windowMs=300000.op=__%.user=%.count
-      String metricNameWithEscSeq = sqlRegex.replace("*", 
"\\*").replace("__%", "..%");
-      javaRegEx = metricNameWithEscSeq.replace("%", ".*");
-    } else {
-      javaRegEx = sqlRegex.replace("%", ".*");
-    }
-    return javaRegEx;
-  }
-
   public void saveHostAggregateRecords(Map<TimelineMetric, 
MetricHostAggregate> hostAggregateMap,
                                        String phoenixTableName) throws 
SQLException {
 
diff --git 
a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsIgniteCache.java
 
b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsIgniteCache.java
index 24fc938..92878cf 100644
--- 
a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsIgniteCache.java
+++ 
b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsIgniteCache.java
@@ -24,8 +24,10 @@ import static 
org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.ti
 import static 
org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_CLUSTER_AGGREGATOR_INTERPOLATION_ENABLED;
 import static 
org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_COLLECTOR_IGNITE_BACKUPS;
 import static 
org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_COLLECTOR_IGNITE_NODES;
+import static 
org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_EVENT_METRIC_PATTERNS;
 import static 
org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRIC_AGGREGATION_SQL_FILTERS;
 import static 
org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_SERVICE_HTTP_POLICY;
+import static 
org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils.getJavaRegexFromSqlRegex;
 import static 
org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils.getRoundedCheckPointTimeMillis;
 import static 
org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils.getTimeSlices;
 import static 
org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils.sliceFromTimelineMetric;
@@ -36,11 +38,15 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.Lock;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import javax.cache.Cache;
 import javax.cache.expiry.CreatedExpiryPolicy;
@@ -114,7 +120,10 @@ public class TimelineMetricsIgniteCache implements 
TimelineMetricDistributedCach
     String filteredMetricPatterns = 
metricConf.get(TIMELINE_METRIC_AGGREGATION_SQL_FILTERS);
     if (!StringUtils.isEmpty(filteredMetricPatterns)) {
       LOG.info("Skipping aggregation for metric patterns : " + 
filteredMetricPatterns);
-      
skipAggrPatternStrings.addAll(Arrays.asList(filteredMetricPatterns.split(",")));
+      for (String patternString : filteredMetricPatterns.split(",")) {
+        String javaPatternString = getJavaRegexFromSqlRegex(patternString);
+        skipAggrPatternStrings.add(javaPatternString);
+      }
     }
 
     if (metricConf.get(TIMELINE_METRICS_COLLECTOR_IGNITE_NODES) != null) {
diff --git 
a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AggregatorUtils.java
 
b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AggregatorUtils.java
index b8338fb..132d447 100644
--- 
a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AggregatorUtils.java
+++ 
b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AggregatorUtils.java
@@ -251,4 +251,18 @@ public class AggregatorUtils {
     long currentTime = System.currentTimeMillis();
     return currentTime - (currentTime % aggregatorPeriod);
   }
+
+  public static String getJavaRegexFromSqlRegex(String sqlRegex) {
+    String javaRegEx;
+    if (sqlRegex.contains("*") || sqlRegex.contains("__%")) {
+      //Special case handling for metric name with * and __%.
+      //For example, dfs.NNTopUserOpCounts.windowMs=300000.op=*.user=%.count
+      // or dfs.NNTopUserOpCounts.windowMs=300000.op=__%.user=%.count
+      String metricNameWithEscSeq = sqlRegex.replace("*", 
"\\*").replace("__%", "..%");
+      javaRegEx = metricNameWithEscSeq.replace("%", ".*");
+    } else {
+      javaRegEx = sqlRegex.replace("%", ".*");
+    }
+    return javaRegEx;
+  }
 }
diff --git 
a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java
 
b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java
index 773e372..15bb852 100644
--- 
a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java
+++ 
b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java
@@ -22,6 +22,7 @@ import static 
org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.ti
 import static 
org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_CLUSTER_AGGREGATOR_INTERPOLATION_ENABLED;
 import static 
org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_EVENT_METRIC_PATTERNS;
 import static 
org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRIC_AGGREGATION_SQL_FILTERS;
+import static 
org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils.getJavaRegexFromSqlRegex;
 import static 
org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils.getTimeSlices;
 import static 
org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils.sliceFromTimelineMetric;
 import static 
org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.GET_METRIC_SQL;
@@ -68,7 +69,6 @@ public class TimelineMetricClusterAggregatorSecond extends 
AbstractTimelineAggre
   protected final boolean interpolationEnabled;
   private TimelineMetricMetadataManager metadataManagerInstance;
   private String skipAggrPatternStrings;
-  private String skipInterpolationMetricPatternStrings;
   private Set<Pattern> skipInterpolationMetricPatterns = new HashSet<>();
   private final static String liveHostsMetricName = "live_hosts";
 
@@ -95,11 +95,11 @@ public class TimelineMetricClusterAggregatorSecond extends 
AbstractTimelineAggre
     this.serverTimeShiftAdjustment = 
Long.parseLong(metricsConf.get(SERVER_SIDE_TIMESIFT_ADJUSTMENT, "90000"));
     this.interpolationEnabled = 
Boolean.parseBoolean(metricsConf.get(TIMELINE_METRICS_CLUSTER_AGGREGATOR_INTERPOLATION_ENABLED,
 "true"));
     this.skipAggrPatternStrings = 
metricsConf.get(TIMELINE_METRIC_AGGREGATION_SQL_FILTERS);
-    this.skipInterpolationMetricPatternStrings = 
metricsConf.get(TIMELINE_METRICS_EVENT_METRIC_PATTERNS, "");
+    String skipInterpolationMetricPatternStrings = 
metricsConf.get(TIMELINE_METRICS_EVENT_METRIC_PATTERNS, "");
 
     if (StringUtils.isNotEmpty(skipInterpolationMetricPatternStrings)) {
       for (String patternString : 
skipInterpolationMetricPatternStrings.split(",")) {
-        String javaPatternString = 
hBaseAccessor.getJavaRegexFromSqlRegex(patternString);
+        String javaPatternString = getJavaRegexFromSqlRegex(patternString);
         LOG.info("SQL pattern " + patternString + " converted to Java pattern 
: " + javaPatternString);
         
skipInterpolationMetricPatterns.add(Pattern.compile(javaPatternString));
       }
diff --git 
a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
 
b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
index 04cf801..f6e0ad2 100644
--- 
a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
+++ 
b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
@@ -750,7 +750,7 @@
   </property>
   <property>
     <name>timeline.metrics.cluster.aggregation.sql.filters</name>
-    <value>sdisk\_%,boottime</value>
+    <value>sdisk\_%,boottime,topology%,</value>
     <description>
       Commas separated list of Metric names or Phoenix 'LIKE' class 
expressions that match metric names
       which prevents certain metrics from being aggregated across hosts.

-- 
To stop receiving notification emails like this one, please contact
avija...@apache.org.

Reply via email to