This is an automated email from the ASF dual-hosted git repository. avijayan pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/ambari.git
commit 4efcf9d9f22131e0afcd1e46de14660f18d2f00e Author: Dmytro Sen <[email protected]> AuthorDate: Tue Jul 11 14:17:58 2017 +0300 AMBARI-21279 Handle scenario when host in-memory aggregation is not working (dsen) --- .../timeline/HBaseTimelineMetricsService.java | 15 +- .../TimelineMetricAggregatorFactory.java | 49 ++++ .../TimelineMetricFilteringHostAggregator.java | 94 ++++++++ .../v2/TimelineMetricFilteringHostAggregator.java | 119 ++++++++++ .../discovery/TimelineMetricMetadataManager.java | 76 +++++-- .../metrics/timeline/query/Condition.java | 2 + .../metrics/timeline/query/DefaultCondition.java | 246 ++++++++++----------- .../metrics/timeline/query/EmptyCondition.java | 11 + .../metrics/timeline/query/PhoenixTransactSQL.java | 13 +- .../query/SplitByMetricNamesCondition.java | 10 + .../metrics/timeline/TestPhoenixTransactSQL.java | 4 +- .../timeline/query/DefaultConditionTest.java | 194 ++++++++++------ 12 files changed, 608 insertions(+), 225 deletions(-) diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricsService.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricsService.java index 2d890c0..4318fd3 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricsService.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricsService.java @@ -82,6 +82,7 @@ public class HBaseTimelineMetricsService extends AbstractService implements Time private static volatile boolean isInitialized = false; private final ScheduledExecutorService watchdogExecutorService = Executors.newSingleThreadScheduledExecutor(); private final Map<AGGREGATOR_NAME, ScheduledExecutorService> scheduledExecutors = new HashMap<>(); + private final ConcurrentHashMap<String, Long> postedAggregatedMap = new ConcurrentHashMap<>(); private TimelineMetricMetadataManager metricMetadataManager; private Integer defaultTopNHostsLimit; private MetricCollectorHAController haController; @@ -172,7 +173,11 @@ public class HBaseTimelineMetricsService extends AbstractService implements Time // Start the minute host aggregator if (Boolean.parseBoolean(metricsConf.get(TIMELINE_METRICS_HOST_INMEMORY_AGGREGATION, "true"))) { - LOG.info("timeline.metrics.host.inmemory.aggregation is set to True, disabling host minute aggregation on collector"); + LOG.info("timeline.metrics.host.inmemory.aggregation is set to True, switching to filtering host minute aggregation on collector"); + TimelineMetricAggregator minuteHostAggregator = + TimelineMetricAggregatorFactory.createFilteringTimelineMetricAggregatorMinute( + hBaseAccessor, metricsConf, metricMetadataManager, haController, postedAggregatedMap); + scheduleAggregatorThread(minuteHostAggregator); } else { TimelineMetricAggregator minuteHostAggregator = TimelineMetricAggregatorFactory.createTimelineMetricAggregatorMinute( @@ -463,8 +468,16 @@ public class HBaseTimelineMetricsService extends AbstractService implements Time @Override public TimelinePutResponse putHostAggregatedMetrics(AggregationResult aggregationResult) throws SQLException, IOException { Map<TimelineMetric, MetricHostAggregate> aggregateMap = new HashMap<>(); + String hostname = null; for (TimelineMetricWithAggregatedValues entry : aggregationResult.getResult()) { aggregateMap.put(entry.getTimelineMetric(), entry.getMetricAggregate()); + hostname = hostname == null ? entry.getTimelineMetric().getHostName() : hostname; + break; + } + long timestamp = aggregationResult.getTimeInMilis(); + postedAggregatedMap.put(hostname, timestamp); + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("Adding host %s to aggregated by in-memory aggregator. Timestamp : %s", hostname, timestamp)); } hBaseAccessor.saveHostAggregateRecords(aggregateMap, PhoenixTransactSQL.METRICS_AGGREGATE_MINUTE_TABLE_NAME); diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAggregatorFactory.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAggregatorFactory.java index 081e610..e90fa84 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAggregatorFactory.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAggregatorFactory.java @@ -23,6 +23,8 @@ import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline. import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.MetricCollectorHAController; import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataManager; +import java.util.concurrent.ConcurrentHashMap; + import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_DAILY_CHECKPOINT_CUTOFF_MULTIPLIER; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_DAILY_DISABLED; @@ -455,4 +457,51 @@ public class TimelineMetricAggregatorFactory { haController ); } + + public static TimelineMetricAggregator createFilteringTimelineMetricAggregatorMinute(PhoenixHBaseAccessor hBaseAccessor, Configuration metricsConf, TimelineMetricMetadataManager metricMetadataManager, MetricCollectorHAController haController, ConcurrentHashMap<String, Long> postedAggregatedMap) { + String checkpointDir = metricsConf.get( + TIMELINE_METRICS_AGGREGATOR_CHECKPOINT_DIR, DEFAULT_CHECKPOINT_LOCATION); + String checkpointLocation = FilenameUtils.concat(checkpointDir, + HOST_AGGREGATE_MINUTE_CHECKPOINT_FILE); + long sleepIntervalMillis = SECONDS.toMillis(metricsConf.getLong + (HOST_AGGREGATOR_MINUTE_SLEEP_INTERVAL, 300l)); // 5 mins + + int checkpointCutOffMultiplier = metricsConf.getInt + (HOST_AGGREGATOR_MINUTE_CHECKPOINT_CUTOFF_MULTIPLIER, 3); + String hostAggregatorDisabledParam = HOST_AGGREGATOR_MINUTE_DISABLED; + + String inputTableName = METRICS_RECORD_TABLE_NAME; + String outputTableName = METRICS_AGGREGATE_MINUTE_TABLE_NAME; + + if (useGroupByAggregator(metricsConf)) { + return new org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.v2.TimelineMetricFilteringHostAggregator( + METRIC_RECORD_MINUTE, + metricMetadataManager, + hBaseAccessor, metricsConf, + checkpointLocation, + sleepIntervalMillis, + checkpointCutOffMultiplier, + hostAggregatorDisabledParam, + inputTableName, + outputTableName, + 120000l, + haController, + postedAggregatedMap + ); + } + + return new TimelineMetricFilteringHostAggregator( + METRIC_RECORD_MINUTE, + metricMetadataManager, + hBaseAccessor, metricsConf, + checkpointLocation, + sleepIntervalMillis, + checkpointCutOffMultiplier, + hostAggregatorDisabledParam, + inputTableName, + outputTableName, + 120000l, + haController, + postedAggregatedMap); + } } diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricFilteringHostAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricFilteringHostAggregator.java new file mode 100644 index 0000000..6e766e9 --- /dev/null +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricFilteringHostAggregator.java @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor; +import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner; +import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.MetricCollectorHAController; +import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataManager; +import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.Condition; +import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.DefaultCondition; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.GET_METRIC_AGGREGATE_ONLY_SQL; + +public class TimelineMetricFilteringHostAggregator extends TimelineMetricHostAggregator { + private static final Log LOG = LogFactory.getLog(TimelineMetricFilteringHostAggregator.class); + private TimelineMetricMetadataManager metricMetadataManager; + private ConcurrentHashMap<String, Long> postedAggregatedMap; + + public TimelineMetricFilteringHostAggregator(AggregationTaskRunner.AGGREGATOR_NAME aggregatorName, + TimelineMetricMetadataManager metricMetadataManager, + PhoenixHBaseAccessor hBaseAccessor, + Configuration metricsConf, + String checkpointLocation, + Long sleepIntervalMillis, + Integer checkpointCutOffMultiplier, + String hostAggregatorDisabledParam, + String tableName, + String outputTableName, + Long nativeTimeRangeDelay, + MetricCollectorHAController haController, + ConcurrentHashMap<String, Long> postedAggregatedMap) { + super(aggregatorName, metricMetadataManager, + hBaseAccessor, metricsConf, + checkpointLocation, + sleepIntervalMillis, + checkpointCutOffMultiplier, + hostAggregatorDisabledParam, + tableName, + outputTableName, + nativeTimeRangeDelay, + haController); + this.metricMetadataManager = metricMetadataManager; + this.postedAggregatedMap = postedAggregatedMap; + } + + @Override + protected Condition prepareMetricQueryCondition(long startTime, long endTime) { + List<String> aggregatedHostnames = new ArrayList<>(); + for (Map.Entry<String, Long> entry : postedAggregatedMap.entrySet()) { + if (entry.getValue() > startTime && entry.getValue() <= endTime) { + aggregatedHostnames.add(entry.getKey()); + } + } + List<String> notAggregatedHostnames = metricMetadataManager.getNotLikeHostnames(aggregatedHostnames); + if (LOG.isDebugEnabled()) { + LOG.debug("Already aggregated hostnames based on postedAggregatedMap : " + aggregatedHostnames); + LOG.debug("Hostnames that will be aggregated : " + notAggregatedHostnames); + } + List<byte[]> uuids = metricMetadataManager.getUuids(new ArrayList<String>(), notAggregatedHostnames, "", ""); + + Condition condition = new DefaultCondition(uuids, null, null, null, null, startTime, + endTime, null, null, true); + condition.setNoLimit(); + condition.setFetchSize(resultsetFetchSize); + condition.setStatement(String.format(GET_METRIC_AGGREGATE_ONLY_SQL, tableName)); + // Retaining order of the row-key avoids client side merge sort. + condition.addOrderByColumn("UUID"); + condition.addOrderByColumn("SERVER_TIME"); + return condition; + } +} diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/v2/TimelineMetricFilteringHostAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/v2/TimelineMetricFilteringHostAggregator.java new file mode 100644 index 0000000..f6e0d8f --- /dev/null +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/v2/TimelineMetricFilteringHostAggregator.java @@ -0,0 +1,119 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.v2; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor; +import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner; +import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.MetricCollectorHAController; +import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataManager; +import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.Condition; +import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.EmptyCondition; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.GET_AGGREGATED_HOST_METRIC_GROUPBY_SQL; + +public class TimelineMetricFilteringHostAggregator extends TimelineMetricHostAggregator { + private TimelineMetricMetadataManager metricMetadataManager; + private ConcurrentHashMap<String, Long> postedAggregatedMap; + + public TimelineMetricFilteringHostAggregator(AggregationTaskRunner.AGGREGATOR_NAME aggregatorName, + TimelineMetricMetadataManager metricMetadataManager, + PhoenixHBaseAccessor hBaseAccessor, + Configuration metricsConf, + String checkpointLocation, + Long sleepIntervalMillis, + Integer checkpointCutOffMultiplier, + String hostAggregatorDisabledParam, + String tableName, + String outputTableName, + Long nativeTimeRangeDelay, + MetricCollectorHAController haController, + ConcurrentHashMap<String, Long> postedAggregatedMap) { + super(aggregatorName, + hBaseAccessor, metricsConf, + checkpointLocation, + sleepIntervalMillis, + checkpointCutOffMultiplier, + hostAggregatorDisabledParam, + tableName, + outputTableName, + nativeTimeRangeDelay, + haController); + this.metricMetadataManager = metricMetadataManager; + this.postedAggregatedMap = postedAggregatedMap; + } + + @Override + protected Condition prepareMetricQueryCondition(long startTime, long endTime) { + List<String> aggregatedHostnames = new ArrayList<>(); + for (Map.Entry<String, Long> entry : postedAggregatedMap.entrySet()) { + if (entry.getValue() > startTime && entry.getValue() <= endTime) { + aggregatedHostnames.add(entry.getKey()); + } + } + List<String> notAggregatedHostnames = metricMetadataManager.getNotLikeHostnames(aggregatedHostnames); + if (LOG.isDebugEnabled()) { + LOG.debug("Already aggregated hostnames based on postedAggregatedMap : " + aggregatedHostnames); + LOG.debug("Hostnames that will be aggregated : " + notAggregatedHostnames); + } + List<byte[]> uuids = metricMetadataManager.getUuids(new ArrayList<String>(), notAggregatedHostnames, "", ""); + + EmptyCondition condition = new EmptyCondition(); + condition.setDoUpdate(true); + + condition.setStatement(String.format(GET_AGGREGATED_HOST_METRIC_GROUPBY_SQL, + outputTableName, endTime, tableName, + getDownsampledMetricSkipClause() + getIncludedUuidsClause(uuids), startTime, endTime)); + + if (LOG.isDebugEnabled()) { + LOG.debug("Condition: " + condition.toString()); + } + + return condition; + } + + private String getIncludedUuidsClause(List<byte[]> uuids) { + StringBuilder sb = new StringBuilder(); + sb.append("("); + + //LIKE clause + // (UUID LIKE ? OR UUID LIKE ?) AND + if (CollectionUtils.isNotEmpty(uuids)) { + for (int i = 0; i < uuids.size(); i++) { + sb.append("UUID"); + sb.append(" LIKE "); + sb.append("'%"); + sb.append(new String(uuids.get(i))); + sb.append("'"); + + if (i == uuids.size() - 1) { + sb.append(") AND "); + } else { + sb.append(" OR "); + } + } + } + return sb.toString(); + } +} diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java index e00c045..bd508c4 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java @@ -69,8 +69,8 @@ public class TimelineMetricMetadataManager { AtomicBoolean SYNC_HOSTED_APPS_METADATA = new AtomicBoolean(false); AtomicBoolean SYNC_HOSTED_INSTANCES_METADATA = new AtomicBoolean(false); private MetricUuidGenStrategy uuidGenStrategy = new HashBasedUuidGenStrategy(); - private static final int timelineMetricUuidLength = 16; - private static final int hostnameUuidLength = 4; + public static final int TIMELINE_METRIC_UUID_LENGTH = 16; + public static final int HOSTNAME_UUID_LENGTH = 4; // Single thread to sync back new writes to the store private final ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(); @@ -344,9 +344,9 @@ public class TimelineMetricMetadataManager { } /** - * Given the hostname, generates a byte array of length 'hostnameUuidLength' + * Given the hostname, generates a byte array of length 'HOSTNAME_UUID_LENGTH' * @param hostname - * @return uuid byte array of length 'hostnameUuidLength' + * @return uuid byte array of length 'HOSTNAME_UUID_LENGTH' */ private byte[] getUuidForHostname(String hostname) { @@ -358,7 +358,7 @@ public class TimelineMetricMetadataManager { } } - byte[] uuid = uuidGenStrategy.computeUuid(hostname, hostnameUuidLength); + byte[] uuid = uuidGenStrategy.computeUuid(hostname, HOSTNAME_UUID_LENGTH); String uuidStr = new String(uuid); if (uuidHostMap.containsKey(uuidStr)) { @@ -379,7 +379,7 @@ public class TimelineMetricMetadataManager { /** * Given a timelineClusterMetric instance, generates a UUID for Metric-App-Instance combination. * @param timelineClusterMetric - * @return uuid byte array of length 'timelineMetricUuidLength' + * @return uuid byte array of length 'TIMELINE_METRIC_UUID_LENGTH' */ public byte[] getUuid(TimelineClusterMetric timelineClusterMetric) { TimelineMetricMetadataKey key = new TimelineMetricMetadataKey(timelineClusterMetric.getMetricName(), @@ -393,7 +393,7 @@ public class TimelineMetricMetadataManager { } } - byte[] uuid = uuidGenStrategy.computeUuid(timelineClusterMetric, timelineMetricUuidLength); + byte[] uuid = uuidGenStrategy.computeUuid(timelineClusterMetric, TIMELINE_METRIC_UUID_LENGTH); String uuidStr = new String(uuid); if (uuidKeyMap.containsKey(uuidStr) && !uuidKeyMap.get(uuidStr).equals(key)) { @@ -419,7 +419,7 @@ public class TimelineMetricMetadataManager { /** * Given a timelineMetric instance, generates a UUID for Metric-App-Instance combination. * @param timelineMetric - * @return uuid byte array of length 'timelineMetricUuidLength' + 'hostnameUuidLength' + * @return uuid byte array of length 'TIMELINE_METRIC_UUID_LENGTH' + 'HOSTNAME_UUID_LENGTH' */ public byte[] getUuid(TimelineMetric timelineMetric) { @@ -433,8 +433,8 @@ public class TimelineMetricMetadataManager { public String getMetricNameFromUuid(byte[] uuid) { byte[] metricUuid = uuid; - if (uuid.length == timelineMetricUuidLength + hostnameUuidLength) { - metricUuid = ArrayUtils.subarray(uuid, 0, timelineMetricUuidLength); + if (uuid.length == TIMELINE_METRIC_UUID_LENGTH + HOSTNAME_UUID_LENGTH) { + metricUuid = ArrayUtils.subarray(uuid, 0, TIMELINE_METRIC_UUID_LENGTH); } TimelineMetricMetadataKey key = uuidKeyMap.get(new String(metricUuid)); @@ -446,11 +446,11 @@ public class TimelineMetricMetadataManager { return null; } - if (uuid.length == timelineMetricUuidLength) { + if (uuid.length == TIMELINE_METRIC_UUID_LENGTH) { TimelineMetricMetadataKey key = uuidKeyMap.get(new String(uuid)); return key != null ? new TimelineMetric(key.metricName, null, key.appId, key.instanceId) : null; } else { - byte[] metricUuid = ArrayUtils.subarray(uuid, 0, timelineMetricUuidLength); + byte[] metricUuid = ArrayUtils.subarray(uuid, 0, TIMELINE_METRIC_UUID_LENGTH); TimelineMetricMetadataKey key = uuidKeyMap.get(new String(metricUuid)); if (key == null) { LOG.error("TimelineMetricMetadataKey is null for : " + Arrays.toString(uuid)); @@ -461,7 +461,7 @@ public class TimelineMetricMetadataManager { timelineMetric.setAppId(key.appId); timelineMetric.setInstanceId(key.instanceId); - byte[] hostUuid = ArrayUtils.subarray(uuid, timelineMetricUuidLength, hostnameUuidLength + timelineMetricUuidLength); + byte[] hostUuid = ArrayUtils.subarray(uuid, TIMELINE_METRIC_UUID_LENGTH, HOSTNAME_UUID_LENGTH + TIMELINE_METRIC_UUID_LENGTH); timelineMetric.setHostName(uuidHostMap.get(new String(hostUuid))); return timelineMetric; } @@ -525,14 +525,23 @@ public class TimelineMetricMetadataManager { appId = appId.toLowerCase(); } if (CollectionUtils.isNotEmpty(sanitizedHostNames)) { - for (String metricName : sanitizedMetricNames) { - TimelineMetric metric = new TimelineMetric(); - metric.setMetricName(metricName); - metric.setAppId(appId); - metric.setInstanceId(instanceId); + if (CollectionUtils.isNotEmpty(sanitizedMetricNames)) { + for (String metricName : sanitizedMetricNames) { + TimelineMetric metric = new TimelineMetric(); + metric.setMetricName(metricName); + metric.setAppId(appId); + metric.setInstanceId(instanceId); + for (String hostname : sanitizedHostNames) { + metric.setHostName(hostname); + byte[] uuid = getUuid(metric); + if (uuid != null) { + uuids.add(uuid); + } + } + } + } else { for (String hostname : sanitizedHostNames) { - metric.setHostName(hostname); - byte[] uuid = getUuid(metric); + byte[] uuid = getUuidForHostname(hostname); if (uuid != null) { uuids.add(uuid); } @@ -554,4 +563,31 @@ public class TimelineMetricMetadataManager { public Map<String, TimelineMetricMetadataKey> getUuidKeyMap() { return uuidKeyMap; } + + public List<String> getNotLikeHostnames(List<String> hostnames) { + List<String> result = new ArrayList<>(); + Set<String> sanitizedHostNames = new HashSet<>(); + if (CollectionUtils.isNotEmpty(hostnames)) { + for (String hostname : hostnames) { + if (hostname.contains("%")) { + String hostRegEx; + hostRegEx = hostname.replace("%", ".*"); + for (String host : HOSTED_APPS_MAP.keySet()) { + if (host.matches(hostRegEx)) { + sanitizedHostNames.add(host); + } + } + } else { + sanitizedHostNames.add(hostname); + } + } + } + + for (String hostname: HOSTED_APPS_MAP.keySet()) { + if (!sanitizedHostNames.contains(hostname)) { + result.add(hostname); + } + } + return result; + } } diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/Condition.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/Condition.java index 9714e1a..4e04e6c 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/Condition.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/Condition.java @@ -46,4 +46,6 @@ public interface Condition { void setNoLimit(); boolean doUpdate(); void setMetricNamesNotCondition(boolean metricNamesNotCondition); + void setHostnamesNotCondition(boolean hostNamesNotCondition); + void setUuidNotCondition(boolean uuidNotCondition); } diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/DefaultCondition.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/DefaultCondition.java index 3c03dca..763e4c7 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/DefaultCondition.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/DefaultCondition.java @@ -19,9 +19,9 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline import org.apache.commons.collections.CollectionUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.metrics2.sink.timeline.TopNConfig; import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor; import org.apache.hadoop.metrics2.sink.timeline.Precision; +import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataManager; import java.util.ArrayList; import java.util.LinkedHashSet; @@ -43,6 +43,8 @@ public class DefaultCondition implements Condition { String statement; Set<String> orderByColumns = new LinkedHashSet<String>(); boolean metricNamesNotCondition = false; + boolean hostNamesNotCondition = false; + boolean uuidNotCondition = false; List<byte[]> uuids = new ArrayList<>(); private static final Log LOG = LogFactory.getLog(DefaultCondition.class); @@ -230,164 +232,142 @@ public class DefaultCondition implements Condition { boolean appendConjunction = false; if (CollectionUtils.isNotEmpty(uuids)) { - // Put a '(' first - sb.append("("); - - //IN clause - // UUID (NOT) IN (?,?,?,?) - if (CollectionUtils.isNotEmpty(uuids)) { - sb.append("UUID"); - if (metricNamesNotCondition) { - sb.append(" NOT"); - } - sb.append(" IN ("); - //Append ?,?,?,? - for (int i = 0; i < uuids.size(); i++) { - sb.append("?"); - if (i < uuids.size() - 1) { - sb.append(", "); - } - } - sb.append(")"); - } - appendConjunction = true; - sb.append(")"); - } - return appendConjunction; - } - - protected boolean appendMetricNameClause(StringBuilder sb) { - boolean appendConjunction = false; - List<String> metricsLike = new ArrayList<>(); - List<String> metricsIn = new ArrayList<>(); - - if (getMetricNames() != null) { - for (String name : getMetricNames()) { - if (name.contains("%")) { - metricsLike.add(name); - } else { - metricsIn.add(name); + List<byte[]> uuidsHost = new ArrayList<>(); + List<byte[]> uuidsMetric = new ArrayList<>(); + List<byte[]> uuidsFull = new ArrayList<>(); + + if (getUuids() != null) { + for (byte[] uuid : uuids) { + if (uuid.length == TimelineMetricMetadataManager.TIMELINE_METRIC_UUID_LENGTH) { + uuidsMetric.add(uuid); + } else if (uuid.length == TimelineMetricMetadataManager.HOSTNAME_UUID_LENGTH) { + uuidsHost.add(uuid); + } else { + uuidsFull.add(uuid); + } } - } - // Put a '(' first - sb.append("("); + // Put a '(' first + sb.append("("); - //IN clause - // METRIC_NAME (NOT) IN (?,?,?,?) - if (CollectionUtils.isNotEmpty(metricsIn)) { - sb.append("METRIC_NAME"); - if (metricNamesNotCondition) { - sb.append(" NOT"); - } - sb.append(" IN ("); - //Append ?,?,?,? - for (int i = 0; i < metricsIn.size(); i++) { - sb.append("?"); - if (i < metricsIn.size() - 1) { - sb.append(", "); + //IN clause + // METRIC_NAME (NOT) IN (?,?,?,?) + if (CollectionUtils.isNotEmpty(uuidsFull)) { + sb.append("UUID"); + if (uuidNotCondition) { + sb.append(" NOT"); + } + sb.append(" IN ("); + //Append ?,?,?,? + for (int i = 0; i < uuidsFull.size(); i++) { + sb.append("?"); + if (i < uuidsFull.size() - 1) { + sb.append(", "); + } } + sb.append(")"); + appendConjunction = true; } - sb.append(")"); - appendConjunction = true; - } - //Put an OR/AND if both types are present - if (CollectionUtils.isNotEmpty(metricsIn) && - CollectionUtils.isNotEmpty(metricsLike)) { - if (metricNamesNotCondition) { + //Put an AND if both types are present + if (CollectionUtils.isNotEmpty(uuidsFull) && + CollectionUtils.isNotEmpty(uuidsMetric)) { sb.append(" AND "); - } else { - sb.append(" OR "); } - } - //LIKE clause - // METRIC_NAME (NOT) LIKE ? OR(AND) METRIC_NAME LIKE ? - if (CollectionUtils.isNotEmpty(metricsLike)) { + // ( for OR + if (!metricNamesNotCondition && uuidsMetric.size() > 1 && (CollectionUtils.isNotEmpty(uuidsFull) || CollectionUtils.isNotEmpty(uuidsHost))) { + sb.append("("); + } - for (int i = 0; i < metricsLike.size(); i++) { - sb.append("METRIC_NAME"); - if (metricNamesNotCondition) { - sb.append(" NOT"); - } - sb.append(" LIKE "); - sb.append("?"); + //LIKE clause for clusterMetric UUIDs + // UUID (NOT) LIKE ? OR(AND) UUID LIKE ? + if (CollectionUtils.isNotEmpty(uuidsMetric)) { - if (i < metricsLike.size() - 1) { + for (int i = 0; i < uuidsMetric.size(); i++) { + sb.append("UUID"); if (metricNamesNotCondition) { - sb.append(" AND "); - } else { - sb.append(" OR "); + sb.append(" NOT"); + } + sb.append(" LIKE "); + sb.append("?"); + + if (i < uuidsMetric.size() - 1) { + if (metricNamesNotCondition) { + sb.append(" AND "); + } else { + sb.append(" OR "); + } + // ) for OR + } else if ((CollectionUtils.isNotEmpty(uuidsFull) || CollectionUtils.isNotEmpty(uuidsHost)) && !metricNamesNotCondition && uuidsMetric.size() > 1) { + sb.append(")"); } } + appendConjunction = true; } - appendConjunction = true; - } - // Finish with a ')' - if (appendConjunction) { - sb.append(")"); - } + //Put an AND if both types are present + if ((CollectionUtils.isNotEmpty(uuidsMetric) || (CollectionUtils.isNotEmpty(uuidsFull) && CollectionUtils.isEmpty(uuidsMetric))) + && CollectionUtils.isNotEmpty(uuidsHost)) { + sb.append(" AND "); + } + // ( for OR + if((CollectionUtils.isNotEmpty(uuidsFull) || CollectionUtils.isNotEmpty(uuidsMetric)) && !hostNamesNotCondition && uuidsHost.size() > 1){ + sb.append("("); + } - metricNames.clear(); - if (CollectionUtils.isNotEmpty(metricsIn)) { - metricNames.addAll(metricsIn); - } - if (CollectionUtils.isNotEmpty(metricsLike)) { - metricNames.addAll(metricsLike); - } - } - return appendConjunction; - } + //LIKE clause for HOST UUIDs + // UUID (NOT) LIKE ? OR(AND) UUID LIKE ? + if (CollectionUtils.isNotEmpty(uuidsHost)) { - protected boolean appendHostnameClause(StringBuilder sb, boolean appendConjunction) { - boolean hostnameContainsRegex = false; - if (hostnames != null) { - for (String hostname : hostnames) { - if (hostname.contains("%")) { - hostnameContainsRegex = true; - break; + for (int i = 0; i < uuidsHost.size(); i++) { + sb.append("UUID"); + if (hostNamesNotCondition) { + sb.append(" NOT"); + } + sb.append(" LIKE "); + sb.append("?"); + + if (i < uuidsHost.size() - 1) { + if (hostNamesNotCondition) { + sb.append(" AND "); + } else { + sb.append(" OR "); + } + // ) for OR + } else if ((CollectionUtils.isNotEmpty(uuidsFull) || CollectionUtils.isNotEmpty(uuidsMetric)) && !hostNamesNotCondition && uuidsHost.size() > 1) { + sb.append(")"); + } + } + appendConjunction = true; } - } - } - StringBuilder hostnamesCondition = new StringBuilder(); - if (hostnameContainsRegex) { - hostnamesCondition.append(" ("); - for (String hostname : getHostnames()) { - if (hostnamesCondition.length() > 2) { - hostnamesCondition.append(" OR "); + // Finish with a ')' + if (appendConjunction) { + sb.append(")"); } - hostnamesCondition.append("HOSTNAME LIKE ?"); - } - hostnamesCondition.append(")"); - - appendConjunction = append(sb, appendConjunction, getHostnames(), hostnamesCondition.toString()); - } else if (hostnames != null && getHostnames().size() > 1) { - for (String hostname : getHostnames()) { - if (hostnamesCondition.length() > 0) { - hostnamesCondition.append(" ,"); - } else { - hostnamesCondition.append(" HOSTNAME IN ("); + + uuids = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(uuidsFull)) { + uuids.addAll(uuidsFull); + } + for (byte[] uuid: uuidsMetric) { + uuids.add(new String(uuid).concat("%").getBytes()); + } + for (byte[] uuid: uuidsHost) { + uuids.add("%".concat(new String(uuid)).getBytes()); } - hostnamesCondition.append('?'); } - hostnamesCondition.append(')'); - appendConjunction = append(sb, appendConjunction, getHostnames(), hostnamesCondition.toString()); - - } else { - appendConjunction = append(sb, appendConjunction, getHostnames(), " HOSTNAME = ?"); } + return appendConjunction; } @Override public String toString() { return "Condition{" + - "metricNames=" + metricNames + - ", hostnames='" + hostnames + '\'' + + "uuids=" + uuids + ", appId='" + appId + '\'' + ", instanceId='" + instanceId + '\'' + ", startTime=" + startTime + @@ -424,6 +404,16 @@ public class DefaultCondition implements Condition { } @Override + public void setHostnamesNotCondition(boolean hostNamesNotCondition) { + this.hostNamesNotCondition = hostNamesNotCondition; + } + + @Override + public void setUuidNotCondition(boolean uuidNotCondition) { + this.uuidNotCondition = uuidNotCondition; + } + + @Override public List<byte[]> getUuids() { return uuids; } diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/EmptyCondition.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/EmptyCondition.java index b667df3..6d43179 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/EmptyCondition.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/EmptyCondition.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query; +import org.apache.commons.lang.NotImplementedException; import org.apache.hadoop.metrics2.sink.timeline.Precision; import java.util.List; @@ -155,4 +156,14 @@ public class EmptyCondition implements Condition { public void setMetricNamesNotCondition(boolean metricNamesNotCondition) { this.metricNamesNotCondition = metricNamesNotCondition; } + + @Override + public void setHostnamesNotCondition(boolean hostNamesNotCondition) { + throw new NotImplementedException("Not implemented"); + } + + @Override + public void setUuidNotCondition(boolean uuidNotCondition) { + throw new NotImplementedException("Not implemented"); + } } diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixTransactSQL.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixTransactSQL.java index d94d14c..2478fb1 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixTransactSQL.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixTransactSQL.java @@ -22,6 +22,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.metrics2.sink.timeline.Precision; import org.apache.hadoop.metrics2.sink.timeline.PrecisionLimitExceededException; import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor; +import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataManager; import java.sql.Connection; import java.sql.PreparedStatement; @@ -194,7 +195,6 @@ public class PhoenixTransactSQL { public static final String UPSERT_CLUSTER_AGGREGATE_TIME_SQL = "UPSERT INTO" + " %s (UUID, SERVER_TIME, " + - "UNITS, " + "METRIC_SUM, " + "METRIC_COUNT, " + "METRIC_MAX, " + @@ -204,7 +204,6 @@ public class PhoenixTransactSQL { public static final String UPSERT_AGGREGATE_RECORD_SQL = "UPSERT INTO " + "%s (UUID, " + "SERVER_TIME, " + - "UNITS, " + "METRIC_SUM, " + "METRIC_MAX, " + "METRIC_MIN," + @@ -775,10 +774,16 @@ public class PhoenixTransactSQL { private static int addUuids(Condition condition, int pos, PreparedStatement stmt) throws SQLException { if (condition.getUuids() != null) { for (int pos2 = 1 ; pos2 <= condition.getUuids().size(); pos2++,pos++) { + byte[] uuid = condition.getUuids().get(pos2 - 1); if (LOG.isDebugEnabled()) { - LOG.debug("Setting pos: " + pos + ", value = " + condition.getUuids().get(pos2 - 1)); + LOG.debug("Setting pos: " + pos + ", value = " + new String(uuid)); + } + + if (uuid.length != TimelineMetricMetadataManager.HOSTNAME_UUID_LENGTH + TimelineMetricMetadataManager.TIMELINE_METRIC_UUID_LENGTH) { + stmt.setString(pos, new String(uuid)); + } else { + stmt.setBytes(pos, uuid); } - stmt.setBytes(pos, condition.getUuids().get(pos2 - 1)); } } return pos; diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/SplitByMetricNamesCondition.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/SplitByMetricNamesCondition.java index 45ea74c..554d2e8 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/SplitByMetricNamesCondition.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/SplitByMetricNamesCondition.java @@ -176,4 +176,14 @@ public class SplitByMetricNamesCondition implements Condition { public void setMetricNamesNotCondition(boolean metricNamesNotCondition) { this.metricNamesNotCondition = metricNamesNotCondition; } + + @Override + public void setHostnamesNotCondition(boolean hostNamesNotCondition) { + + } + + @Override + public void setUuidNotCondition(boolean uuidNotCondition) { + + } } diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestPhoenixTransactSQL.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestPhoenixTransactSQL.java index cb3f3a7..fe801ad 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestPhoenixTransactSQL.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestPhoenixTransactSQL.java @@ -567,7 +567,7 @@ public class TestPhoenixTransactSQL { String conditionClause = condition.getConditionClause().toString(); String expectedClause = " UUID IN (" + "SELECT UUID FROM METRIC_RECORD WHERE " + - "(UUID IN (?, ?)) AND " + + "(UUID LIKE ? OR UUID LIKE ?) AND " + "SERVER_TIME >= ? AND SERVER_TIME < ? " + "GROUP BY UUID ORDER BY MAX(METRIC_MAX) DESC LIMIT 2) AND SERVER_TIME >= ? AND SERVER_TIME < ?"; @@ -585,7 +585,7 @@ public class TestPhoenixTransactSQL { String conditionClause = condition.getConditionClause().toString(); String expectedClause = " UUID IN (" + "SELECT UUID FROM METRIC_RECORD WHERE " + - "(UUID IN (?, ?, ?)) AND " + + "(UUID LIKE ? OR UUID LIKE ? OR UUID LIKE ?) AND " + "SERVER_TIME >= ? AND SERVER_TIME < ? " + "GROUP BY UUID ORDER BY MAX(METRIC_MAX) DESC LIMIT 2) AND SERVER_TIME >= ? AND SERVER_TIME < ?"; diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/DefaultConditionTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/DefaultConditionTest.java index e4e9225..71226e8 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/DefaultConditionTest.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/DefaultConditionTest.java @@ -29,88 +29,142 @@ public class DefaultConditionTest { @Test public void testMetricNameWhereCondition() { - List<String> metricNames = new ArrayList<>(); - - //Only IN clause. - - metricNames.add("M1"); - DefaultCondition condition = new DefaultCondition(metricNames,null,null,null,null,null,null,null,true); + //EMPTY + List<byte[]> uuids = new ArrayList<>(); + DefaultCondition condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true); StringBuilder sb = new StringBuilder(); - condition.appendMetricNameClause(sb); - Assert.assertEquals(sb.toString(), "(METRIC_NAME IN (?))"); - Assert.assertTrue(CollectionUtils.isEqualCollection(metricNames, condition.getMetricNames())); + condition.appendUuidClause(sb); + Assert.assertEquals(sb.toString(), ""); + Assert.assertTrue(CollectionUtils.isEqualCollection(uuids, condition.getUuids())); - metricNames.add("m2"); - condition = new DefaultCondition(metricNames,null,null,null,null,null,null,null,true); + //Metric uuid + uuids.add(new byte[16]); + condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true); sb = new StringBuilder(); - condition.appendMetricNameClause(sb); - Assert.assertEquals(sb.toString(), "(METRIC_NAME IN (?, ?))"); - Assert.assertTrue(CollectionUtils.isEqualCollection(metricNames, condition.getMetricNames())); - - // Only NOT IN clause - condition = new DefaultCondition(metricNames,null,null,null,null,null,null,null,true); - condition.setMetricNamesNotCondition(true); + condition.appendUuidClause(sb); + Assert.assertEquals(sb.toString(), "(UUID LIKE ?)"); + Assert.assertEquals(uuids.size(), condition.getUuids().size()); + Assert.assertTrue(new String(condition.getUuids().get(0)).endsWith("%")); + + //metric uuid + Host uuid + uuids.add(new byte[4]); + condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true); sb = new StringBuilder(); - condition.appendMetricNameClause(sb); - Assert.assertEquals(sb.toString(), "(METRIC_NAME NOT IN (?, ?))"); - Assert.assertTrue(CollectionUtils.isEqualCollection(metricNames, condition.getMetricNames())); - - metricNames.clear(); - - //Only LIKE clause - metricNames.add("disk%"); - condition = new DefaultCondition(metricNames,null,null,null,null,null,null,null,true); + condition.appendUuidClause(sb); + Assert.assertEquals(sb.toString(), "(UUID LIKE ? AND UUID LIKE ?)"); + Assert.assertEquals(uuids.size(), condition.getUuids().size()); + Assert.assertTrue(new String(condition.getUuids().get(1)).startsWith("%")); + + //metric + host + full + uuids.add(new byte[20]); + uuids.add(new byte[20]); + condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true); sb = new StringBuilder(); - condition.appendMetricNameClause(sb); - Assert.assertEquals(sb.toString(), "(METRIC_NAME LIKE ?)"); - Assert.assertTrue(CollectionUtils.isEqualCollection(metricNames, condition.getMetricNames())); + condition.appendUuidClause(sb); + Assert.assertEquals(sb.toString(), "(UUID IN (?, ?) AND UUID LIKE ? AND UUID LIKE ?)"); + Assert.assertEquals(uuids.size(), condition.getUuids().size()); - metricNames.add("cpu%"); - condition = new DefaultCondition(metricNames,null,null,null,null,null,null,null,true); + //Only IN clause. + uuids.clear(); + uuids.add(new byte[20]); + condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true); + sb = new StringBuilder(); + condition.appendUuidClause(sb); + Assert.assertEquals(sb.toString(), "(UUID IN (?))"); + Assert.assertEquals(uuids.size(), condition.getUuids().size()); + + //metric NOT LIKE + uuids.clear(); + uuids.add(new byte[16]); + condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true); sb = new StringBuilder(); - condition.appendMetricNameClause(sb); - Assert.assertEquals(sb.toString(), "(METRIC_NAME LIKE ? OR METRIC_NAME LIKE ?)"); - Assert.assertTrue(CollectionUtils.isEqualCollection(metricNames, condition.getMetricNames())); - - //Only NOT LIKE clause - condition = new DefaultCondition(metricNames,null,null,null,null,null,null,null,true); condition.setMetricNamesNotCondition(true); + condition.appendUuidClause(sb); + Assert.assertEquals(sb.toString(), "(UUID NOT LIKE ?)"); + Assert.assertEquals(uuids.size(), condition.getUuids().size()); + + //metric NOT LIKE host LIKE + uuids.clear(); + uuids.add(new byte[16]); + uuids.add(new byte[4]); + condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true); sb = new StringBuilder(); - condition.appendMetricNameClause(sb); - Assert.assertEquals(sb.toString(), "(METRIC_NAME NOT LIKE ? AND METRIC_NAME NOT LIKE ?)"); - Assert.assertTrue(CollectionUtils.isEqualCollection(metricNames, condition.getMetricNames())); - - metricNames.clear(); - - // IN followed by LIKE clause - metricNames.add("M1"); - metricNames.add("disk%"); - metricNames.add("M2"); - condition = new DefaultCondition(metricNames,null,null,null,null,null,null,null,true); + condition.setMetricNamesNotCondition(true); + condition.appendUuidClause(sb); + Assert.assertEquals(sb.toString(), "(UUID NOT LIKE ? AND UUID LIKE ?)"); + Assert.assertEquals(uuids.size(), condition.getUuids().size()); + Assert.assertTrue(new String(condition.getUuids().get(0)).endsWith("%")); + Assert.assertTrue(new String(condition.getUuids().get(1)).startsWith("%")); + + //metric LIKE host NOT LIKE + uuids.clear(); + uuids.add(new byte[16]); + uuids.add(new byte[4]); + condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true); sb = new StringBuilder(); - condition.appendMetricNameClause(sb); - Assert.assertEquals(sb.toString(), "(METRIC_NAME IN (?, ?) OR METRIC_NAME LIKE ?)"); - Assert.assertEquals(metricNames.get(2), "disk%"); - - metricNames.clear(); - //NOT IN followed by NOT LIKE clause - metricNames.add("disk%"); - metricNames.add("metric1"); - metricNames.add("cpu%"); - condition = new DefaultCondition(metricNames,null,null,null,null,null,null,null,true); + condition.setHostnamesNotCondition(true); + condition.appendUuidClause(sb); + Assert.assertEquals(sb.toString(), "(UUID LIKE ? AND UUID NOT LIKE ?)"); + Assert.assertEquals(uuids.size(), condition.getUuids().size()); + Assert.assertTrue(new String(condition.getUuids().get(0)).endsWith("%")); + Assert.assertTrue(new String(condition.getUuids().get(1)).startsWith("%")); + + //metric LIKE or LIKE host LIKE + uuids.clear(); + uuids.add(new byte[4]); + uuids.add(new byte[16]); + uuids.add(new byte[16]); + condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true); sb = new StringBuilder(); - condition.setMetricNamesNotCondition(true); - condition.appendMetricNameClause(sb); - Assert.assertEquals(sb.toString(), "(METRIC_NAME NOT IN (?) AND METRIC_NAME NOT LIKE ? AND METRIC_NAME NOT LIKE ?)"); - Assert.assertEquals(metricNames.get(0), "metric1"); - - //Empty - metricNames.clear(); - condition = new DefaultCondition(metricNames,null,null,null,null,null,null,null,true); + condition.appendUuidClause(sb); + Assert.assertEquals(sb.toString(), "((UUID LIKE ? OR UUID LIKE ?) AND UUID LIKE ?)"); + Assert.assertEquals(uuids.size(), condition.getUuids().size()); + Assert.assertTrue(new String(condition.getUuids().get(0)).endsWith("%")); + Assert.assertTrue(new String(condition.getUuids().get(1)).endsWith("%")); + Assert.assertTrue(new String(condition.getUuids().get(2)).startsWith("%")); + + //UUID in metric LIKE or LIKE host LIKE + uuids.clear(); + uuids.add(new byte[16]); + uuids.add(new byte[16]); + uuids.add(new byte[20]); + uuids.add(new byte[4]); + condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true); sb = new StringBuilder(); - condition.appendMetricNameClause(sb); - Assert.assertEquals(sb.toString(), ""); - + condition.appendUuidClause(sb); + Assert.assertEquals(sb.toString(), "(UUID IN (?) AND (UUID LIKE ? OR UUID LIKE ?) AND UUID LIKE ?)"); + Assert.assertEquals(uuids.size(), condition.getUuids().size()); + Assert.assertTrue(new String(condition.getUuids().get(1)).endsWith("%")); + Assert.assertTrue(new String(condition.getUuids().get(2)).endsWith("%")); + Assert.assertTrue(new String(condition.getUuids().get(3)).startsWith("%")); + + //metric LIKE host LIKE or LIKE + uuids.clear(); + uuids.add(new byte[16]); + uuids.add(new byte[4]); + uuids.add(new byte[4]); + condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true); + sb = new StringBuilder(); + condition.appendUuidClause(sb); + Assert.assertEquals(sb.toString(), "(UUID LIKE ? AND (UUID LIKE ? OR UUID LIKE ?))"); + Assert.assertEquals(uuids.size(), condition.getUuids().size()); + Assert.assertTrue(new String(condition.getUuids().get(0)).endsWith("%")); + Assert.assertTrue(new String(condition.getUuids().get(1)).startsWith("%")); + Assert.assertTrue(new String(condition.getUuids().get(2)).startsWith("%")); + + //UUID NOT IN metric LIKE host LIKE + uuids.clear(); + uuids.add(new byte[20]); + uuids.add(new byte[16]); + uuids.add(new byte[4]); + condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true); + sb = new StringBuilder(); + condition.setUuidNotCondition(true); + condition.appendUuidClause(sb); + Assert.assertEquals(sb.toString(), "(UUID NOT IN (?) AND UUID LIKE ? AND UUID LIKE ?)"); + Assert.assertEquals(uuids.size(), condition.getUuids().size()); + Assert.assertTrue(new String(condition.getUuids().get(1)).endsWith("%")); + Assert.assertTrue(new String(condition.getUuids().get(2)).startsWith("%")); } } -- To stop receiving notification emails like this one, please contact [email protected].
