This is an automated email from the ASF dual-hosted git repository. avijayan pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/ambari.git
commit 7d215e0bdc519f450a127b6adbf11438c23136ad Author: Aravindan Vijayan <[email protected]> AuthorDate: Fri May 11 07:03:29 2018 -0700 Revert "AMBARI-23804 : Refine AMS HBase region splitting calculation based on UUID work (Refactor Split point computation. Allow only Murmur3Hash)." This reverts commit d4d2767ae0d9c8e3043144df52bd5d6094b61171. --- .../metrics/core/timeline/PhoenixHBaseAccessor.java | 1 - .../timeline/TimelineMetricSplitPointComputer.java | 19 +++++++++---------- .../discovery/TimelineMetricMetadataManager.java | 20 +++++++++++++++++++- .../timeline/uuid/Murmur3HashUuidGenStrategy.java | 13 +------------ .../TimelineMetricSplitPointComputerTest.java | 21 +++++++++------------ 5 files changed, 38 insertions(+), 36 deletions(-) diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java index c684b0a..dec7850 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java @@ -492,7 +492,6 @@ public class PhoenixHBaseAccessor { PreparedStatement pStmt = null; TimelineMetricSplitPointComputer splitPointComputer = new TimelineMetricSplitPointComputer( metricsConf, hbaseConf, metadataManagerInstance); - splitPointComputer.computeSplitPoints(); String encoding = metricsConf.get(HBASE_ENCODING_SCHEME, DEFAULT_ENCODING); String compression = metricsConf.get(HBASE_COMPRESSION_SCHEME, DEFAULT_TABLE_COMPRESSION); diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricSplitPointComputer.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricSplitPointComputer.java index 05366cc..89bb843 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricSplitPointComputer.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricSplitPointComputer.java @@ -49,11 +49,6 @@ public class TimelineMetricSplitPointComputer { private static final int SLAVE_EQUIDISTANT_POINTS = 50; private static final int MASTER_EQUIDISTANT_POINTS = 5; - private double hbaseTotalHeapsize; - private double hbaseMemstoreUpperLimit; - private double hbaseMemstoreFlushSize; - private TimelineMetricMetadataManager timelineMetricMetadataManager = null; - private List<byte[]> precisionSplitPoints = new ArrayList<>(); private List<byte[]> aggregateSplitPoints = new ArrayList<>(); @@ -71,14 +66,18 @@ public class TimelineMetricSplitPointComputer { slaveComponents.addAll(Arrays.asList(componentsString.split(","))); } - this.timelineMetricMetadataManager = timelineMetricMetadataManager; - hbaseTotalHeapsize = metricsConf.getDouble("hbase_total_heapsize", 1024*1024*1024); - hbaseMemstoreUpperLimit = hbaseConf.getDouble("hbase.regionserver.global.memstore.upperLimit", 0.3); - hbaseMemstoreFlushSize = hbaseConf.getDouble("hbase.hregion.memstore.flush.size", 134217728); + double hbaseTotalHeapsize = metricsConf.getDouble("hbase_total_heapsize", 1024*1024*1024); + double hbaseMemstoreUpperLimit = hbaseConf.getDouble("hbase.regionserver.global.memstore.upperLimit", 0.5); + double hbaseMemstoreFlushSize = hbaseConf.getDouble("hbase.hregion.memstore.flush.size", 134217728); + + computeSplitPoints(hbaseTotalHeapsize, hbaseMemstoreUpperLimit, hbaseMemstoreFlushSize, timelineMetricMetadataManager); } - protected void computeSplitPoints() { + private void computeSplitPoints(double hbaseTotalHeapsize, + double hbaseMemstoreUpperLimit, + double hbaseMemstoreFlushSize, + TimelineMetricMetadataManager timelineMetricMetadataManager) { double memstoreMaxMemory = hbaseMemstoreUpperLimit * hbaseTotalHeapsize; int maxInMemoryRegions = (int) ((memstoreMaxMemory / hbaseMemstoreFlushSize) - OTHER_TABLE_STATIC_REGIONS); diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/discovery/TimelineMetricMetadataManager.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/discovery/TimelineMetricMetadataManager.java index 86226ec..737c2ff 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/discovery/TimelineMetricMetadataManager.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/discovery/TimelineMetricMetadataManager.java @@ -397,6 +397,18 @@ public class TimelineMetricMetadataManager { } } + if (!HOSTED_APPS_MAP.isEmpty()) { + Map.Entry<String, TimelineMetricHostMetadata> entry = HOSTED_APPS_MAP.entrySet().iterator().next(); + TimelineMetricHostMetadata timelineMetricHostMetadata = entry.getValue(); + if (timelineMetricHostMetadata.getUuid() != null && timelineMetricHostMetadata.getUuid().length == 16) { + HOSTNAME_UUID_LENGTH = 16; + uuidGenStrategy = new MD5UuidGenStrategy(); + } else { + HOSTNAME_UUID_LENGTH = 4; + uuidGenStrategy = new Murmur3HashUuidGenStrategy(); + } + } + for (String host : HOSTED_APPS_MAP.keySet()) { TimelineMetricHostMetadata timelineMetricHostMetadata = HOSTED_APPS_MAP.get(host); if (timelineMetricHostMetadata != null && timelineMetricHostMetadata.getUuid() != null) { @@ -411,7 +423,13 @@ public class TimelineMetricMetadataManager { * @return the UUID generator of type org.apache.ambari.metrics.core.timeline.uuid.MetricUuidGenStrategy */ private MetricUuidGenStrategy getUuidStrategy(Configuration configuration) { - return new Murmur3HashUuidGenStrategy(); + String strategy = configuration.get(TIMELINE_METRICS_UUID_GEN_STRATEGY, ""); + if ("md5".equalsIgnoreCase(strategy)){ + return new MD5UuidGenStrategy(); + } else { + //Default + return new Murmur3HashUuidGenStrategy(); + } } /** diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/Murmur3HashUuidGenStrategy.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/Murmur3HashUuidGenStrategy.java index af8cee5..9418aa4 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/Murmur3HashUuidGenStrategy.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/Murmur3HashUuidGenStrategy.java @@ -24,12 +24,7 @@ import org.apache.commons.lang.StringUtils; public class Murmur3HashUuidGenStrategy implements MetricUuidGenStrategy{ - /** - * Compute Murmur3Hash 16 byte UUID for a Metric-App-Instance. - * @param timelineClusterMetric input metric - * @param maxLength Max length of returned UUID. (Will always be 16 for this technique) - * @return 16 byte UUID. - */ @Override + @Override public byte[] computeUuid(TimelineClusterMetric timelineClusterMetric, int maxLength) { String metricString = timelineClusterMetric.getMetricName() + timelineClusterMetric.getAppId(); @@ -40,12 +35,6 @@ public class Murmur3HashUuidGenStrategy implements MetricUuidGenStrategy{ return Hashing.murmur3_128().hashBytes(metricBytes).asBytes(); } - /** - * Compute Murmur3Hash 4 byte UUID for a String. - * @param value String input - * @param maxLength Max length of returned UUID. (Will always be 4 for this technique) - * @return 4 byte UUID. - */ @Override public byte[] computeUuid(String value, int maxLength) { byte[] valueBytes = value.getBytes(); diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricSplitPointComputerTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricSplitPointComputerTest.java index 150dac2..4d663cc 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricSplitPointComputerTest.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricSplitPointComputerTest.java @@ -51,7 +51,7 @@ public class TimelineMetricSplitPointComputerTest { expect(metricsConfMock.getDouble("hbase_total_heapsize", 1024*1024*1024)).andReturn(1024 * 1024 * 1024.0).once(); Configuration hbaseConfMock = EasyMock.createMock(Configuration.class); - expect(hbaseConfMock.getDouble("hbase.regionserver.global.memstore.upperLimit", 0.3)).andReturn(0.3).once(); + expect(hbaseConfMock.getDouble("hbase.regionserver.global.memstore.upperLimit", 0.5)).andReturn(0.5).once(); expect(hbaseConfMock.getDouble("hbase.hregion.memstore.flush.size", 134217728)).andReturn(134217728.0).once(); TimelineMetricMetadataManager metricMetadataManagerMock = EasyMock.createNiceMock(TimelineMetricMetadataManager.class); @@ -62,7 +62,6 @@ public class TimelineMetricSplitPointComputerTest { TimelineMetricSplitPointComputer timelineMetricSplitPointComputer = new TimelineMetricSplitPointComputer(metricsConfMock, hbaseConfMock, metricMetadataManagerMock); - timelineMetricSplitPointComputer.computeSplitPoints(); Assert.assertEquals(timelineMetricSplitPointComputer.getPrecisionSplitPoints().size(), 3); Assert.assertEquals(timelineMetricSplitPointComputer.getClusterAggregateSplitPoints().size(), 1); @@ -86,7 +85,7 @@ public class TimelineMetricSplitPointComputerTest { expect(metricsConfMock.getDouble("hbase_total_heapsize", 1024*1024*1024)).andReturn(8589934592.0).once(); Configuration hbaseConfMock = EasyMock.createMock(Configuration.class); - expect(hbaseConfMock.getDouble("hbase.regionserver.global.memstore.upperLimit", 0.3)).andReturn(0.3).once(); + expect(hbaseConfMock.getDouble("hbase.regionserver.global.memstore.upperLimit", 0.5)).andReturn(0.5).once(); expect(hbaseConfMock.getDouble("hbase.hregion.memstore.flush.size", 134217728)).andReturn(134217728.0).once(); TimelineMetricMetadataManager metricMetadataManagerMock = EasyMock.createNiceMock(TimelineMetricMetadataManager.class); @@ -97,11 +96,10 @@ public class TimelineMetricSplitPointComputerTest { TimelineMetricSplitPointComputer timelineMetricSplitPointComputer = new TimelineMetricSplitPointComputer(metricsConfMock, hbaseConfMock, metricMetadataManagerMock); - timelineMetricSplitPointComputer.computeSplitPoints(); - Assert.assertEquals(timelineMetricSplitPointComputer.getPrecisionSplitPoints().size(), 6); - Assert.assertEquals(timelineMetricSplitPointComputer.getClusterAggregateSplitPoints().size(), 1); - Assert.assertEquals(timelineMetricSplitPointComputer.getHostAggregateSplitPoints().size(), 1); + Assert.assertEquals(timelineMetricSplitPointComputer.getPrecisionSplitPoints().size(), 16); + Assert.assertEquals(timelineMetricSplitPointComputer.getClusterAggregateSplitPoints().size(), 3); + Assert.assertEquals(timelineMetricSplitPointComputer.getHostAggregateSplitPoints().size(), 3); } @Test @@ -121,7 +119,7 @@ public class TimelineMetricSplitPointComputerTest { expect(metricsConfMock.getDouble("hbase_total_heapsize", 1024*1024*1024)).andReturn(24 * 1024 * 1024 * 1024.0).once(); Configuration hbaseConfMock = EasyMock.createMock(Configuration.class); - expect(hbaseConfMock.getDouble("hbase.regionserver.global.memstore.upperLimit", 0.3)).andReturn(0.3).once(); + expect(hbaseConfMock.getDouble("hbase.regionserver.global.memstore.upperLimit", 0.5)).andReturn(0.5).once(); expect(hbaseConfMock.getDouble("hbase.hregion.memstore.flush.size", 134217728)).andReturn(2 * 134217728.0).once(); TimelineMetricMetadataManager metricMetadataManagerMock = EasyMock.createNiceMock(TimelineMetricMetadataManager.class); @@ -132,10 +130,9 @@ public class TimelineMetricSplitPointComputerTest { TimelineMetricSplitPointComputer timelineMetricSplitPointComputer = new TimelineMetricSplitPointComputer(metricsConfMock, hbaseConfMock, metricMetadataManagerMock); - timelineMetricSplitPointComputer.computeSplitPoints(); - Assert.assertEquals(timelineMetricSplitPointComputer.getPrecisionSplitPoints().size(), 14); - Assert.assertEquals(timelineMetricSplitPointComputer.getClusterAggregateSplitPoints().size(), 3); - Assert.assertEquals(timelineMetricSplitPointComputer.getHostAggregateSplitPoints().size(), 3); + Assert.assertEquals(timelineMetricSplitPointComputer.getPrecisionSplitPoints().size(), 28); + Assert.assertEquals(timelineMetricSplitPointComputer.getClusterAggregateSplitPoints().size(), 6); + Assert.assertEquals(timelineMetricSplitPointComputer.getHostAggregateSplitPoints().size(), 6); } } -- To stop receiving notification emails like this one, please contact [email protected].
