Repository: ambari Updated Branches: refs/heads/trunk bb4046625 -> da8b308ea
AMBARI-17726. ArrayIndexOutOfBounds exception thrown on shard calculation. (swagle) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/da8b308e Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/da8b308e Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/da8b308e Branch: refs/heads/trunk Commit: da8b308ea85d1a9755274c2b377f601e7e2913e4 Parents: bb40466 Author: Siddharth Wagle <[email protected]> Authored: Thu Jul 14 22:36:41 2016 -0700 Committer: Siddharth Wagle <[email protected]> Committed: Fri Jul 15 13:22:30 2016 -0700 ---------------------------------------------------------------------- .../timeline/AbstractTimelineMetricsSink.java | 48 ++++++++++---------- ...icSinkWriteShardHostnameHashingStrategy.java | 5 +- .../availability/ShardingStrategyTest.java | 12 +++++ .../MetricCollectorHAController.java | 2 +- 4 files changed, 39 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/da8b308e/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java index f0cea7a..426eb42 100644 --- a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java +++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java @@ -326,33 +326,31 @@ public abstract class AbstractTimelineMetricsSink { } } - String configuredCollectors = getConfiguredCollectors(); + String collectorHosts = getConfiguredCollectors(); // Reach out to all configured collectors before Zookeeper - if (configuredCollectors != null && !configuredCollectors.isEmpty()) { - String collectorHosts = getConfiguredCollectors(); - if (!collectorHosts.isEmpty()) { - String[] hosts = collectorHosts.split(","); - for (String hostPortStr : hosts) { - if (hostPortStr != null && !hostPortStr.isEmpty()) { - String[] hostPortPair = hostPortStr.split(":"); - if (hostPortPair.length < 2) { - LOG.warn("Collector port is missing from the configuration."); - continue; - } - String hostStr = hostPortPair[0].trim(); - String portStr = hostPortPair[1].trim(); - // Check liveliness and get known instances - try { - Collection<String> liveHosts = findLiveCollectorHostsFromKnownCollector(hostStr, portStr); - // Update live Hosts - current host will already be a part of this - for (String host : liveHosts) { - allKnownLiveCollectors.add(host); - } - } catch (MetricCollectorUnavailableException e) { - allKnownLiveCollectors.remove(hostStr); - LOG.info("Collector " + hostStr + " is not longer live. Removing " + - "it from list of know live collector hosts : " + allKnownLiveCollectors); + if (collectorHosts != null && !collectorHosts.isEmpty()) { + String[] hosts = collectorHosts.split(","); + for (String hostPortStr : hosts) { + if (hostPortStr != null && !hostPortStr.isEmpty()) { + String[] hostPortPair = hostPortStr.split(":"); + if (hostPortPair.length < 2) { + LOG.warn("Collector port is missing from the configuration."); + continue; + } + String hostStr = hostPortPair[0].trim(); + String portStr = hostPortPair[1].trim(); + // Check liveliness and get known instances + try { + Collection<String> liveHosts = findLiveCollectorHostsFromKnownCollector(hostStr, portStr); + // Update live Hosts - current host will already be a part of this + for (String host : liveHosts) { + allKnownLiveCollectors.add(host); } + break; // Found at least 1 live collector + } catch (MetricCollectorUnavailableException e) { + LOG.info("Collector " + hostStr + " is not longer live. Removing " + + "it from list of know live collector hosts : " + allKnownLiveCollectors); + allKnownLiveCollectors.remove(hostStr); } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/da8b308e/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricSinkWriteShardHostnameHashingStrategy.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricSinkWriteShardHostnameHashingStrategy.java b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricSinkWriteShardHostnameHashingStrategy.java index 1c89884..25bff54 100644 --- a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricSinkWriteShardHostnameHashingStrategy.java +++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricSinkWriteShardHostnameHashingStrategy.java @@ -36,8 +36,9 @@ public class MetricSinkWriteShardHostnameHashingStrategy implements MetricSinkWr @Override public String findCollectorShard(List<String> collectorHosts) { - int index = (int) (hostnameHash % collectorHosts.size()); - String collectorHost = collectorHosts.get(index); + long index = hostnameHash % collectorHosts.size(); + index = index < 0 ? index + collectorHosts.size() : index; + String collectorHost = collectorHosts.get((int) index); LOG.info(String.format("Calculated collector shard %s based on hostname: %s", collectorHost, hostname)); return collectorHost; } http://git-wip-us.apache.org/repos/asf/ambari/blob/da8b308e/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/ShardingStrategyTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/ShardingStrategyTest.java b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/ShardingStrategyTest.java index c6041db..60ed824 100644 --- a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/ShardingStrategyTest.java +++ b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/ShardingStrategyTest.java @@ -49,4 +49,16 @@ public class ShardingStrategyTest { Assert.assertEquals("mycollector-1.hostname.domain", collectorShard1); Assert.assertEquals("mycollector-2.hostname.domain", collectorShard2); } + + @Test + public void testShardStrategyOnOverflow() { + List<String> collectorHosts = new ArrayList<String>() {{ + add("ambari-sid-4.c.pramod-thangali.internal"); + add("ambari-sid-5.c.pramod-thangali.internal"); + }}; + + MetricSinkWriteShardStrategy strategy = new MetricSinkWriteShardHostnameHashingStrategy("ambari-sid-4.c.pramod-thangali.internal"); + String collector = strategy.findCollectorShard(collectorHosts); + Assert.assertTrue(collector != null && !collector.isEmpty()); + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/da8b308e/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAController.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAController.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAController.java index 84e4153..edce367 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAController.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAController.java @@ -47,7 +47,7 @@ import static org.apache.helix.model.IdealState.RebalanceMode.FULL_AUTO; public class MetricCollectorHAController { private static final Log LOG = LogFactory.getLog(MetricCollectorHAController.class); - static final String CLUSTER_NAME = "ambari-metrics-cluster-unsecure"; + static final String CLUSTER_NAME = "ambari-metrics-cluster"; static final String METRIC_AGGREGATORS = "METRIC_AGGREGATORS"; static final String DEFAULT_STATE_MODEL = "OnlineOffline"; static final String INSTANCE_NAME_DELIMITER = "_";
