http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/uuid/TimelineMetricUuidManagerTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/uuid/TimelineMetricUuidManagerTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/uuid/TimelineMetricUuidManagerTest.java new file mode 100644 index 0000000..65db9c8 --- /dev/null +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/uuid/TimelineMetricUuidManagerTest.java @@ -0,0 +1,175 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.metrics.core.timeline.uuid; + +import org.apache.ambari.metrics.core.timeline.aggregators.TimelineClusterMetric; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.URL; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class TimelineMetricUuidManagerTest { + + + private List<String> apps = Arrays.asList("namenode", + "datanode", "master_hbase", "slave_hbase", "kafka_broker", "nimbus", "ams-hbase", + "accumulo", "nodemanager", "resourcemanager", "ambari_server", "HOST", "timeline_metric_store_watcher", + "jobhistoryserver", "hiveserver2", "hivemetastore", "applicationhistoryserver", "amssmoketestfake"); + + private Map<String, Set<String>> metricSet = new HashMap<>(populateMetricWhitelistFromFile()); + + @Test + public void testHashBasedUuidForMetricName() throws SQLException { + + MetricUuidGenStrategy strategy = new HashBasedUuidGenStrategy(); + Map<String, TimelineClusterMetric> uuids = new HashMap<>(); + for (String app : metricSet.keySet()) { + Set<String> metrics = metricSet.get(app); + for (String metric : metrics) { + TimelineClusterMetric timelineClusterMetric = new TimelineClusterMetric(metric, app, null, -1l); + byte[] uuid = strategy.computeUuid(timelineClusterMetric, 16); + Assert.assertNotNull(uuid); + Assert.assertTrue(uuid.length == 16); + String uuidStr = new String(uuid); + Assert.assertFalse(uuids.containsKey(uuidStr) && !uuids.containsValue(timelineClusterMetric)); + if (uuids.containsKey(uuidStr) ) { + if (!uuids.containsValue(timelineClusterMetric)) { + System.out.println("COLLISION : " + timelineClusterMetric.toString() + " = " + uuids.get(uuidStr)); + } + } + uuids.put(uuidStr, timelineClusterMetric); + } + } + } + + @Test + public void testHaseBasedUuidForAppIds() throws SQLException { + + MetricUuidGenStrategy strategy = new HashBasedUuidGenStrategy(); + Map<String, TimelineClusterMetric> uuids = new HashMap<>(); + for (String app : metricSet.keySet()) { + TimelineClusterMetric timelineClusterMetric = new TimelineClusterMetric("TestMetric", app, null, -1l); + byte[] uuid = strategy.computeUuid(timelineClusterMetric, 16); + String uuidStr = new String(uuid); + if (uuids.containsKey(uuidStr) ) { + if (!uuids.containsValue(timelineClusterMetric)) { + System.out.println("COLLISION : " + timelineClusterMetric.toString() + " = " + uuids.get(uuidStr)); + } + } + uuids.put(uuidStr, timelineClusterMetric); + } + } + + @Test + public void testHashBasedUuidForHostnames() throws SQLException { + + MetricUuidGenStrategy strategy = new HashBasedUuidGenStrategy(); + Map<String, String> uuids = new HashMap<>(); + + List<String> hosts = new ArrayList<>(); + String hostPrefix = "TestHost."; + String hostSuffix = ".ambari.apache.org"; + + for (int i=0; i<=2000; i++) { + hosts.add(hostPrefix + i + hostSuffix); + } + + for (String host : hosts) { + byte[] uuid = strategy.computeUuid(host, 4); + Assert.assertNotNull(uuid); + Assert.assertTrue(uuid.length == 4); + String uuidStr = new String(uuid); + Assert.assertFalse(uuids.containsKey(uuidStr)); + uuids.put(uuidStr, host); + } + } + + + @Test + public void testRandomUuidForWhitelistedMetrics() throws SQLException { + + MetricUuidGenStrategy strategy = new RandomUuidGenStrategy(); + Map<String, String> uuids = new HashMap<>(); + for (String app : metricSet.keySet()) { + Set<String> metrics = metricSet.get(app); + for (String metric : metrics) { + byte[] uuid = strategy.computeUuid(new TimelineClusterMetric(metric, app, null, -1l), 16); + Assert.assertNotNull(uuid); + Assert.assertTrue(uuid.length == 16); + String uuidStr = new String(uuid); + Assert.assertFalse(uuids.containsKey(uuidStr) && !uuids.containsValue(metric)); + uuids.put(uuidStr, metric); + } + } + } + + public Map<String, Set<String>> populateMetricWhitelistFromFile() { + + + Map<String, Set<String>> metricSet = new HashMap<String, Set<String>>(); + FileInputStream fstream = null; + BufferedReader br = null; + String strLine; + for (String appId : apps) { + URL fileUrl = ClassLoader.getSystemResource("metrics_def/" + appId.toUpperCase() + ".dat"); + + Set<String> metricsForApp = new HashSet<>(); + try { + fstream = new FileInputStream(fileUrl.getPath()); + br = new BufferedReader(new InputStreamReader(fstream)); + while ((strLine = br.readLine()) != null) { + strLine = strLine.trim(); + metricsForApp.add(strLine); + } + } catch (Exception ioEx) { + System.out.println("Metrics for AppId " + appId + " not found."); + } finally { + if (br != null) { + try { + br.close(); + } catch (IOException e) { + } + } + + if (fstream != null) { + try { + fstream.close(); + } catch (IOException e) { + } + } + } + metricsForApp.add("live_hosts"); + metricSet.put(appId.contains("hbase") ? "hbase" : appId, metricsForApp); + System.out.println("Found " + metricsForApp.size() + " metrics for appId = " + appId); + } + return metricSet; + } +}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/timeline/TestGenericObjectMapper.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/timeline/TestGenericObjectMapper.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/timeline/TestGenericObjectMapper.java new file mode 100644 index 0000000..4a1bd5a --- /dev/null +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/timeline/TestGenericObjectMapper.java @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.metrics.timeline; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.io.WritableComparator; +import org.apache.ambari.metrics.timeline.GenericObjectMapper; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + [email protected] [email protected] +public class TestGenericObjectMapper { + + @Test + public void testEncoding() { + testEncoding(Long.MAX_VALUE); + testEncoding(Long.MIN_VALUE); + testEncoding(0l); + testEncoding(128l); + testEncoding(256l); + testEncoding(512l); + testEncoding(-256l); + } + + private static void testEncoding(long l) { + byte[] b = GenericObjectMapper.writeReverseOrderedLong(l); + assertEquals("error decoding", l, + GenericObjectMapper.readReverseOrderedLong(b, 0)); + byte[] buf = new byte[16]; + System.arraycopy(b, 0, buf, 5, 8); + assertEquals("error decoding at offset", l, + GenericObjectMapper.readReverseOrderedLong(buf, 5)); + if (l > Long.MIN_VALUE) { + byte[] a = GenericObjectMapper.writeReverseOrderedLong(l-1); + assertEquals("error preserving ordering", 1, + WritableComparator.compareBytes(a, 0, a.length, b, 0, b.length)); + } + if (l < Long.MAX_VALUE) { + byte[] c = GenericObjectMapper.writeReverseOrderedLong(l+1); + assertEquals("error preserving ordering", 1, + WritableComparator.compareBytes(b, 0, b.length, c, 0, c.length)); + } + } + + private static void verify(Object o) throws IOException { + assertEquals(o, GenericObjectMapper.read(GenericObjectMapper.write(o))); + } + + @Test + public void testValueTypes() throws IOException { + verify(Integer.MAX_VALUE); + verify(Integer.MIN_VALUE); + assertEquals(Integer.MAX_VALUE, GenericObjectMapper.read( + GenericObjectMapper.write((long) Integer.MAX_VALUE))); + assertEquals(Integer.MIN_VALUE, GenericObjectMapper.read( + GenericObjectMapper.write((long) Integer.MIN_VALUE))); + verify((long)Integer.MAX_VALUE + 1l); + verify((long)Integer.MIN_VALUE - 1l); + + verify(Long.MAX_VALUE); + verify(Long.MIN_VALUE); + + assertEquals(42, GenericObjectMapper.read(GenericObjectMapper.write(42l))); + verify(42); + verify(1.23); + verify("abc"); + verify(true); + List<String> list = new ArrayList<String>(); + list.add("123"); + list.add("abc"); + verify(list); + Map<String,String> map = new HashMap<String,String>(); + map.put("k1","v1"); + map.put("k2","v2"); + verify(map); + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/webapp/TestTimelineWebServices.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/webapp/TestTimelineWebServices.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/webapp/TestTimelineWebServices.java new file mode 100644 index 0000000..3456af6 --- /dev/null +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/webapp/TestTimelineWebServices.java @@ -0,0 +1,118 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.metrics.webapp; + +import static org.junit.Assert.assertEquals; + +import javax.ws.rs.core.MediaType; + +import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics; +import org.apache.ambari.metrics.core.timeline.TestTimelineMetricStore; +import org.apache.ambari.metrics.core.timeline.TimelineMetricStore; +import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; +import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; +import org.junit.Test; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.servlet.GuiceServletContextListener; +import com.google.inject.servlet.ServletModule; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.api.client.config.DefaultClientConfig; +import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; +import com.sun.jersey.test.framework.JerseyTest; +import com.sun.jersey.test.framework.WebAppDescriptor; + +import junit.framework.Assert; + + +public class TestTimelineWebServices extends JerseyTest { + private static TimelineMetricStore metricStore; + private long beforeTime; + + private Injector injector = Guice.createInjector(new ServletModule() { + + @Override + protected void configureServlets() { + bind(YarnJacksonJaxbJsonProvider.class); + bind(TimelineWebServices.class); + bind(GenericExceptionHandler.class); + try { + metricStore = new TestTimelineMetricStore(); + } catch (Exception e) { + Assert.fail(); + } + bind(TimelineMetricStore.class).toInstance(metricStore); + serve("/*").with(GuiceContainer.class); + } + + }); + + public class GuiceServletConfig extends GuiceServletContextListener { + + @Override + protected Injector getInjector() { + return injector; + } + } + + public TestTimelineWebServices() { + super(new WebAppDescriptor.Builder( + "org.apache.ambari.metrics.webapp") + .contextListenerClass(GuiceServletConfig.class) + .filterClass(com.google.inject.servlet.GuiceFilter.class) + .contextPath("jersey-guice-filter") + .servletPath("/") + .clientConfig(new DefaultClientConfig(YarnJacksonJaxbJsonProvider.class)) + .build()); + } + + @Test + public void testAbout() throws Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("timeline") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + TimelineWebServices.AboutInfo about = + response.getEntity(TimelineWebServices.AboutInfo.class); + Assert.assertNotNull(about); + Assert.assertEquals("AMS API", about.getAbout()); + } + + private static void verifyMetrics(TimelineMetrics metrics) { + Assert.assertNotNull(metrics); + Assert.assertEquals("cpu_user", metrics.getMetrics().get(0).getMetricName()); + Assert.assertEquals(3, metrics.getMetrics().get(0).getMetricValues().size()); + Assert.assertEquals("mem_free", metrics.getMetrics().get(1).getMetricName()); + Assert.assertEquals(3, metrics.getMetrics().get(1).getMetricValues().size()); + } + + @Test + public void testGetMetrics() throws Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("timeline") + .path("metrics").queryParam("metricNames", "cpu_user").queryParam("precision", "seconds") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + verifyMetrics(response.getEntity(TimelineMetrics.class)); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/TestAppMetrics.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/TestAppMetrics.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/TestAppMetrics.java deleted file mode 100644 index 499dab6..0000000 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/TestAppMetrics.java +++ /dev/null @@ -1,134 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.data; - -import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.util.Json; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; - -import static org.junit.Assert.assertEquals; - -public class TestAppMetrics { - private static final String SAMPLE_SINGLE_METRIC_HOST_JSON = "{\n" + - " \"metrics\" : [ {\n" + - " \"instanceid\" : \"\",\n" + - " \"hostname\" : \"localhost\",\n" + - " \"metrics\" : {\n" + - " \"0\" : \"5.35\",\n" + - " \"5000\" : \"5.35\",\n" + - " \"10000\" : \"5.35\",\n" + - " \"15000\" : \"5.35\"\n" + - " },\n" + - " \"starttime\" : \"1411663170112\",\n" + - " \"appid\" : \"HOST\",\n" + - " \"metricname\" : \"disk_free\"\n" + - " } ]\n" + - "}"; - - private static final String SAMPLE_TWO_METRIC_HOST_JSON = "{\n" + - " \"metrics\" : [ {\n" + - " \"instanceid\" : \"\",\n" + - " \"hostname\" : \"localhost\",\n" + - " \"metrics\" : {\n" + - " \"0\" : \"5.35\",\n" + - " \"5000\" : \"5.35\",\n" + - " \"10000\" : \"5.35\",\n" + - " \"15000\" : \"5.35\"\n" + - " },\n" + - " \"starttime\" : \"0\",\n" + - " \"appid\" : \"HOST\",\n" + - " \"metricname\" : \"disk_free\"\n" + - " }, {\n" + - " \"instanceid\" : \"\",\n" + - " \"hostname\" : \"localhost\",\n" + - " \"metrics\" : {\n" + - " \"0\" : \"94.0\",\n" + - " \"5000\" : \"94.0\",\n" + - " \"10000\" : \"94.0\",\n" + - " \"15000\" : \"94.0\"\n" + - " },\n" + - " \"starttime\" : \"0\",\n" + - " \"appid\" : \"HOST\",\n" + - " \"metricname\" : \"mem_cached\"\n" + - " } ]\n" + - "}"; - - private long[] timestamps; - - @Before - public void setUp() throws Exception { - timestamps = new long[4]; - timestamps[0] = 0; - timestamps[1] = timestamps[0] + 5000; - timestamps[2] = timestamps[1] + 5000; - timestamps[3] = timestamps[2] + 5000; - - } - - @Test - public void testHostDiskMetricsSerialization() throws IOException { - long timestamp = 1411663170112L; - AppMetrics appMetrics = new AppMetrics(new ApplicationInstance("localhost", AppID.HOST, ""), timestamp); - - Metric diskFree = appMetrics.createMetric("disk_free"); - double value = 5.35; - - diskFree.putMetric(timestamps[0], Double.toString(value)); - diskFree.putMetric(timestamps[1], Double.toString(value)); - diskFree.putMetric(timestamps[2], Double.toString(value)); - diskFree.putMetric(timestamps[3], Double.toString(value)); - - appMetrics.addMetric(diskFree); - - String expected = SAMPLE_SINGLE_METRIC_HOST_JSON; - String s = new Json(true).serialize(appMetrics); - - assertEquals("Serialized Host Metrics", expected, s); - } - - - @Test - public void testSingleHostManyMetricsSerialization() throws IOException { - AppMetrics appMetrics = new AppMetrics(new ApplicationInstance("localhost", AppID.HOST, ""), timestamps[0]); - - Metric diskFree = appMetrics.createMetric("disk_free"); - double value = 5.35; - diskFree.putMetric(timestamps[0], Double.toString(value)); - diskFree.putMetric(timestamps[1], Double.toString(value)); - diskFree.putMetric(timestamps[2], Double.toString(value)); - diskFree.putMetric(timestamps[3], Double.toString(value)); - - appMetrics.addMetric(diskFree); - - Metric memCache = appMetrics.createMetric("mem_cached"); - double memVal = 94; - memCache.putMetric(timestamps[0], Double.toString(memVal)); - memCache.putMetric(timestamps[1], Double.toString(memVal)); - memCache.putMetric(timestamps[2], Double.toString(memVal)); - memCache.putMetric(timestamps[3], Double.toString(memVal)); - - appMetrics.addMetric(memCache); - - String expected = SAMPLE_TWO_METRIC_HOST_JSON; - String s = new Json(true).serialize(appMetrics); - - assertEquals("Serialized Host Metrics", expected, s); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/TestMetric.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/TestMetric.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/TestMetric.java deleted file mode 100644 index a0572a2..0000000 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/TestMetric.java +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.data; - -import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.util.Json; -import org.junit.Test; - -import java.io.IOException; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.entry; -import static org.junit.Assert.assertEquals; - -public class TestMetric { - private static final String SAMPLE_METRIC_IN_JSON = "{\n" + - " \"instanceid\" : \"\",\n" + - " \"hostname\" : \"localhost\",\n" + - " \"metrics\" : {\n" + - " \"0\" : \"5.35\",\n" + - " \"5000\" : \"5.35\",\n" + - " \"10000\" : \"5.35\",\n" + - " \"15000\" : \"5.35\"\n" + - " },\n" + - " \"starttime\" : \"0\",\n" + - " \"appid\" : \"HOST\",\n" + - " \"metricname\" : \"disk_free\"\n" + - "}"; - - @Test - public void testSerializeToJson() throws IOException { - Metric diskOnHostMetric = new Metric(new ApplicationInstance("localhost", AppID.HOST, ""), "disk_free", 0); - - long timestamp = 0; - double value = 5.35; - - diskOnHostMetric.putMetric(timestamp, Double.toString(value)); - diskOnHostMetric.putMetric(timestamp + 5000, Double.toString(value)); - diskOnHostMetric.putMetric(timestamp + 10000, Double.toString(value)); - diskOnHostMetric.putMetric(timestamp + 15000, Double.toString(value)); - - String expected = SAMPLE_METRIC_IN_JSON; - String s = new Json(true).serialize(diskOnHostMetric); - - assertEquals("Json should match", expected, s); - } - - @Test - public void testDeserializeObjectFromString() throws IOException { - String source = SAMPLE_METRIC_IN_JSON; - - Metric m = new Json().deserialize(source, Metric.class); - - assertEquals("localhost", m.getHostname()); - assertEquals("HOST", m.getAppid()); - assertEquals("", m.getInstanceid()); - assertEquals("disk_free", m.getMetricname()); - assertEquals("0", m.getStarttime()); - - assertThat(m.getMetrics()).isNotEmpty().hasSize(4).contains( - entry("0", "5.35"), - entry("5000", "5.35"), - entry("10000", "5.35"), - entry("15000", "5.35")); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/AMSJMeterLoadTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/AMSJMeterLoadTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/AMSJMeterLoadTest.java deleted file mode 100644 index 7f168f2..0000000 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/AMSJMeterLoadTest.java +++ /dev/null @@ -1,198 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.jmetertest.jmetertest; - -import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.MetricsLoadSimulator; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedReader; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; - -public class AMSJMeterLoadTest { - - private final static Logger LOG = LoggerFactory.getLogger(AMSJMeterLoadTest.class); - private static String PROPERTIES_FILE = "loadsimulator/ams-jmeter.properties"; - private ScheduledExecutorService scheduledExecutorService = null; - private List<AppGetMetric> appGetMetrics; - private Properties amsJmeterProperties = null; - - public AMSJMeterLoadTest(Map<String, String> args) { - - String testType = args.get("type"); - String userDefinedPropertiesFile = args.get("amsJmeterPropertiesFile"); - if (null == userDefinedPropertiesFile || userDefinedPropertiesFile.isEmpty()) { - this.amsJmeterProperties = readProperties(PROPERTIES_FILE); - } else { - this.amsJmeterProperties = readProperties(userDefinedPropertiesFile); - } - - if ("U".equals(testType)) { //GET metrics simulator - int numInstances = Integer.valueOf(amsJmeterProperties.getProperty("num-ui-instances")); - this.scheduledExecutorService = Executors.newScheduledThreadPool(numInstances); - this.appGetMetrics = initializeGetMetricsPayload(amsJmeterProperties); - this.runTest(numInstances); - } else { //PUT Metrics simulator - Map<String, String> mapArgs = new HashMap<String, String>(); - mapArgs.put("hostName", (args.get("host-prefix") != null) ? args.get("host-prefix") : amsJmeterProperties.getProperty("host-prefix")); - mapArgs.put("minHostIndex", (args.get("min-host-index") != null) ? args.get("min-host-index") : amsJmeterProperties.getProperty("min-host-index")); - mapArgs.put("numberOfHosts", (args.get("num-hosts") != null) ? args.get("num-hosts") : amsJmeterProperties.getProperty("num-hosts")); - mapArgs.put("metricsHostName", (args.get("ams-host-port") != null) ? args.get("ams-host-port") : amsJmeterProperties.getProperty("ams-host-port")); - mapArgs.put("collectInterval", (args.get("collection-interval") != null) ? args.get("collection-interval") : amsJmeterProperties.getProperty("collection-interval")); - mapArgs.put("sendInterval", (args.get("send-interval") != null) ? args.get("send-interval") : amsJmeterProperties.getProperty("send-interval")); - mapArgs.put("master", (args.get("create-master") != null) ? args.get("create-master") : amsJmeterProperties.getProperty("create-master")); - System.out.println("AMS Load Simulation Parameters : " + mapArgs); - MetricsLoadSimulator.startTest(mapArgs); - } - } - - public static Properties readProperties(String propertiesFile) { - try { - Properties properties = new Properties(); - InputStream inputStream = ClassLoader.getSystemResourceAsStream(propertiesFile); - if (inputStream == null) { - inputStream = new FileInputStream(propertiesFile); - } - properties.load(inputStream); - return properties; - } catch (IOException ioEx) { - LOG.error("Error reading properties file for jmeter"); - return null; - } - } - - private static List<GetMetricRequestInfo> readMetricsFromFile(String app) { - InputStream input = null; - List<GetMetricRequestInfo> metricList = new ArrayList<>(); - String fileName = "ui_metrics_def/" + app + ".dat"; - - try { - input = ClassLoader.getSystemResourceAsStream(fileName); - BufferedReader reader = new BufferedReader(new InputStreamReader(input)); - String line; - List<String> metrics = new ArrayList<>(); - while ((line = reader.readLine()) != null) { - - if (line.startsWith("|")) { - boolean needsTimestamps = line.contains("startTime"); - boolean needsHost = line.contains("hostname"); - metricList.add(new GetMetricRequestInfo(metrics, needsTimestamps, needsHost)); - metrics.clear(); - } else { - metrics.add(line); - } - } - return metricList; - } catch (IOException e) { - LOG.error("Cannot read file " + fileName + " for appID " + app, e); - } finally { - if (input != null) { - try { - input.close(); - } catch (IOException ex) { - } - } - } - return null; - } - - private static List<AppGetMetric> initializeGetMetricsPayload(Properties amsJmeterProperties) { - - List<AppGetMetric> appGetMetrics = new ArrayList<AppGetMetric>(); - String appsToTest = amsJmeterProperties.getProperty("apps-to-test"); - String[] apps; - - if (appsToTest != null && !appsToTest.isEmpty()) { - apps = StringUtils.split(appsToTest, ","); - } else { - apps = new String[JmeterTestPlanTask.ClientApp.values().length]; - int ctr = 0; - for (JmeterTestPlanTask.ClientApp app : JmeterTestPlanTask.ClientApp.values()) - apps[ctr++] = app.getId(); - } - - for (String app : apps) { - - int interval = Integer.valueOf(amsJmeterProperties.getProperty("get-interval")); - String intervalString = amsJmeterProperties.getProperty(app + "-get-interval"); - if (intervalString != null && !intervalString.isEmpty()) { - interval = Integer.valueOf(intervalString); - } - appGetMetrics.add(new AppGetMetric(readMetricsFromFile(app), interval, app)); - } - - return appGetMetrics; - } - - public void runTest(int numInstances) { - - int appRefreshRate = Integer.valueOf(amsJmeterProperties.getProperty("app-refresh-rate")); - for (int i = 0; i < numInstances; i++) { - ScheduledFuture future = scheduledExecutorService.scheduleAtFixedRate(new JmeterTestPlanTask(appGetMetrics, - amsJmeterProperties), 0, appRefreshRate, TimeUnit.MILLISECONDS); - } - } - - /** - * Sample Usage: - * java -cp "lib/*":ambari-metrics-timelineservice-2.1.1.0.jar org.apache.hadoop.yarn.server.applicationhistoryservice - * .metrics.loadsimulator.jmeter.AMSJMeterLoadTest - * -t UI -p ambari-metrics-timelineservice/src/main/resources/jmeter/ams-jmeter.properties - */ - public static void main(String[] args) { - Map<String, String> mapArgs = parseArgs(args); - new AMSJMeterLoadTest(mapArgs); - } - - private static Map<String, String> parseArgs(String[] args) { - Map<String, String> mapProps = new HashMap<String, String>(); - if (args.length == 0) { - printUsage(); - throw new RuntimeException("Unexpected argument, See usage message."); - } else { - for (int i = 0; i < args.length; i += 2) { - String arg = args[i]; - mapProps.put(arg.substring(1), args[i+1]); - } - } - return mapProps; - } - - public static void printUsage() { - System.err.println("Usage: java AMSJmeterLoadTest [OPTIONS]"); - System.err.println("Options: "); - System.err.println("[--t type (S=>Sink/U=>UI)] [-ams-host-port localhost:6188] [-min-host-index 2] [-host-prefix TestHost.] [-num-hosts 2] " + - "[-create-master true] [-collection-interval 10000 ] [-send-interval 60000 ] [-p amsJmeterPropertiesFile (Optional)]"); - } - -} - - http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/AppGetMetric.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/AppGetMetric.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/AppGetMetric.java deleted file mode 100644 index a12cb4a..0000000 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/AppGetMetric.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.jmetertest.jmetertest; - -import java.util.List; - -public class AppGetMetric { - - private String app; - private int interval; - private List<GetMetricRequestInfo> requests; - - public AppGetMetric(List<GetMetricRequestInfo> requests, int interval, String app) { - this.setMetricRequests(requests); - this.setInterval(interval); - this.setApp(app); - } - - public List<GetMetricRequestInfo> getMetricRequests() { - return requests; - } - - public void setMetricRequests(List<GetMetricRequestInfo> requests) { - this.requests = requests; - } - - public int getInterval() { - return interval; - } - - public void setInterval(int interval) { - this.interval = interval; - } - - public String getApp() { - return app; - } - - public void setApp(String app) { - this.app = app; - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/GetMetricRequestInfo.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/GetMetricRequestInfo.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/GetMetricRequestInfo.java deleted file mode 100644 index 4bd44d9..0000000 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/GetMetricRequestInfo.java +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.jmetertest.jmetertest; - -import org.apache.commons.lang.StringUtils; - -import java.util.List; - - -public class GetMetricRequestInfo { - - private String metricStringPayload; - private boolean needsTimestamps; - private boolean needsHost; - - public GetMetricRequestInfo(List<String> metrics, boolean needsTimestamps, boolean needsHost) { - - this.setMetricStringPayload(StringUtils.join(metrics, ",")); - this.setNeedsTimestamps(needsTimestamps); - this.setNeedsHost(needsHost); - } - - public String getMetricStringPayload() { - return metricStringPayload; - } - - public void setMetricStringPayload(String metricStringPayload) { - this.metricStringPayload = metricStringPayload; - } - - public boolean needsTimestamps() { - return needsTimestamps; - } - - public void setNeedsTimestamps(boolean needsTimestamps) { - this.needsTimestamps = needsTimestamps; - } - - public boolean needsHost() { - return needsHost; - } - - public void setNeedsHost(boolean needsHost) { - this.needsHost = needsHost; - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/JmeterTestPlanTask.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/JmeterTestPlanTask.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/JmeterTestPlanTask.java deleted file mode 100644 index c6df162..0000000 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/JmeterTestPlanTask.java +++ /dev/null @@ -1,276 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.jmetertest.jmetertest; - -import org.apache.commons.io.IOUtils; -import org.apache.jmeter.control.LoopController; -import org.apache.jmeter.engine.StandardJMeterEngine; -import org.apache.jmeter.protocol.http.sampler.HTTPSampler; -import org.apache.jmeter.protocol.http.util.HTTPConstants; -import org.apache.jmeter.reporters.ResultCollector; -import org.apache.jmeter.reporters.Summariser; -import org.apache.jmeter.testelement.TestElement; -import org.apache.jmeter.testelement.TestPlan; -import org.apache.jmeter.threads.JMeterContextService; -import org.apache.jmeter.threads.ThreadGroup; -import org.apache.jmeter.timers.ConstantTimer; -import org.apache.jmeter.util.JMeterUtils; -import org.apache.jorphan.collections.HashTree; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.InputStream; -import java.io.OutputStream; -import java.lang.reflect.Field; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Random; - -public class JmeterTestPlanTask implements Runnable { - - private static StandardJMeterEngine jmeterEngine = null; - private final static Logger LOG = LoggerFactory.getLogger(JmeterTestPlanTask.class); - private List<AppGetMetric> appGetMetrics; - private Properties amsJmeterProperties; - private HashTree amsTestPlanTree; - private TestPlan amsTestPlan; - private static final String JMETER_HOME = "loadsimulator"; - private static final String JMETER_PROPERTIES_FILE = JMETER_HOME + "/jmeter.properties"; - private static final String SAVESERVICE_PROPERTIES_FILE = JMETER_HOME + "/saveservice.properties"; - - public enum ClientApp { - HOST("HOST"), - NAMENODE("NAMENODE"), - HBASE("HBASE"), - NIMBUS("NIMBUS"), - KAFKA_BROKER("KAFKA_BROKER"), - FLUME_HANDLER("FLUME_HANDLER"), - AMS_HBASE("AMS-HBASE"), - NODEMANAGER("NODEMANAGER"), - RESOURCEMANAGER("RESOURCEMANAGER"), - DATANODE("DATANODE"); - - private String id; - - private ClientApp(String id) { - this.id = id; - } - - public String getId() { - return id; - } - } - - public JmeterTestPlanTask(List<AppGetMetric> appGetMetrics, Properties amsJmeterProperties) { - this.appGetMetrics = appGetMetrics; - this.amsJmeterProperties = amsJmeterProperties; - amsTestPlanTree = new HashTree(); - amsTestPlan = new TestPlan("AMS JMeter Load Test plan"); - System.out.println("Starting AMS Jmeter load testing"); - } - - public void run() { - if (jmeterEngine != null) { - - Object[] threadGroups = amsTestPlanTree.getArray(amsTestPlan); - for (Object threadGroupObj : threadGroups) { - if (threadGroupObj instanceof ThreadGroup) { - ThreadGroup threadGroup = (ThreadGroup) threadGroupObj; - threadGroup.stop(); - } - } - amsTestPlanTree.clear(); - jmeterEngine.askThreadsToStop(); - jmeterEngine.stopTest(); - JMeterContextService.endTest(); - } - - //Start the new test plan for the new app. - try { - //Initialize Jmeter essentials - jmeterEngine = new StandardJMeterEngine(); - JMeterContextService.getContext().setEngine(jmeterEngine); - - //Workaround to supply JMeterUtils with jmeter.prooperties from JAR. - JMeterUtils.setJMeterHome(""); - Field f = new JMeterUtils().getClass().getDeclaredField("appProperties"); - f.setAccessible(true); - f.set(null, AMSJMeterLoadTest.readProperties(JMETER_PROPERTIES_FILE)); - - //Copy saveservices.properties file to tmp dir for JMeter to consume. - InputStream inputStream = ClassLoader.getSystemResourceAsStream(SAVESERVICE_PROPERTIES_FILE); - if (inputStream == null) { - inputStream = new FileInputStream(SAVESERVICE_PROPERTIES_FILE); - } - String tmpDir = System.getProperty("java.io.tmpdir"); - OutputStream outputStream = new FileOutputStream(tmpDir + "/saveservice.properties"); - IOUtils.copy(inputStream, outputStream); - outputStream.close(); - JMeterUtils.setProperty("saveservice_properties", tmpDir + "/saveservice.properties"); - - //Initialize Test plan - amsTestPlan.setProperty(TestElement.TEST_CLASS, TestPlan.class.getName()); - amsTestPlanTree.add("AMS Test plan", amsTestPlan); - - //Choose a random APP to run the perform GET metrics request. - int currentAppIndex = new Random().nextInt(appGetMetrics.size()); - - //Create ThreadGroup for the App - createThreadGroupHashTree(currentAppIndex, amsJmeterProperties, amsTestPlanTree, amsTestPlan); - - //Geneates the JMX file that you can use through the GUI mode. - //SaveService.saveTree(amsTestPlanTree, new FileOutputStream(JMETER_HOME + "/" + "amsTestPlan.jmx")); - - //Summarizer output to get test progress in stdout like. - Summariser summariser = null; - String summariserName = JMeterUtils.getPropDefault("summariser.name", "summary"); - if (summariserName.length() > 0) { - summariser = new Summariser(summariserName); - } - - //Store execution results into a .jtl file - String jmeterLogFile = tmpDir + "/amsJmeterTestResults.jtl"; - ResultCollector resultCollector = new ResultCollector(summariser); - resultCollector.setFilename(jmeterLogFile); - amsTestPlanTree.add(amsTestPlanTree.getArray()[0], resultCollector); - jmeterEngine.configure(amsTestPlanTree); - jmeterEngine.run(); - - LOG.info("AMS Jmeter Test started up successfully"); - - } catch (Exception ioEx) { - amsTestPlanTree.clear(); - jmeterEngine.askThreadsToStop(); - jmeterEngine.stopTest(); - JMeterContextService.endTest(); - LOG.error("Error occurred while running AMS load test : " + ioEx.getMessage()); - ioEx.printStackTrace(); - } - } - - private ConstantTimer createConstantTimer(int delay) { - ConstantTimer timer = new ConstantTimer(); - timer.setDelay("" + delay); - return timer; - } - - private Map<String, String> getAppSpecificParameters(String app, GetMetricRequestInfo request, Properties amsJmeterProperties) { - - Map<String, String> parametersMap = new HashMap<String, String>(); - String hostPrefix = amsJmeterProperties.getProperty("host-prefix"); - String hostSuffix = amsJmeterProperties.getProperty("host-suffix"); - int minHostIndex = Integer.valueOf(amsJmeterProperties.getProperty("min-host-index")); - int numHosts = Integer.valueOf(amsJmeterProperties.getProperty("num-hosts")); - - parametersMap.put("appId", app); - - if (request.needsTimestamps()) { - long currentTime = System.currentTimeMillis(); - long oneHourBack = currentTime - 3600 * 1000; - parametersMap.put("startTime", String.valueOf(oneHourBack)); - parametersMap.put("endTime", String.valueOf(currentTime)); - } - - if (request.needsHost()) { - if (ClientApp.AMS_HBASE.getId().equals(app)) { - parametersMap.put("hostname", amsJmeterProperties.getProperty("ams-host")); - } else if (ClientApp.HOST.getId().equals(app) || ClientApp.NODEMANAGER.getId().equals(app)) { - int randomHost = minHostIndex + new Random().nextInt(numHosts); - parametersMap.put("hostname", hostPrefix + randomHost + hostSuffix); - } else { - parametersMap.put("hostname", hostPrefix + amsJmeterProperties.getProperty(app + "-host") + hostSuffix); - } - } - parametersMap.put("metricNames", request.getMetricStringPayload()); - return parametersMap; - } - - private void createThreadGroupHashTree(int appIndex, Properties amsJmeterProperties, HashTree amsTestPlanTree, TestPlan amsTestPlan) { - - AppGetMetric appGetMetric = appGetMetrics.get(appIndex); - String app = appGetMetric.getApp(); - int interval = appGetMetric.getInterval(); - - //Read and validate AMS information. - String[] amsHostPort = amsJmeterProperties.getProperty("ams-host-port").split(":"); - String amsHost = amsHostPort[0]; - String amsPath = amsJmeterProperties.getProperty("ams-path"); - int amsPort = Integer.valueOf(amsHostPort[1]); - int numLoops = Integer.valueOf(amsJmeterProperties.getProperty("num-get-calls-per-app")); - - LoopController loopController = createLoopController(app + " GET loop controller", numLoops, false); - for (GetMetricRequestInfo request : appGetMetric.getMetricRequests()) { - - ThreadGroup threadGroup = createThreadGroup(app + " GET threadGroup", 1, 0, loopController); - - HashTree threadGroupHashTree = amsTestPlanTree.add(amsTestPlan, threadGroup); - Map<String, String> parametersMap = getAppSpecificParameters(app, request, amsJmeterProperties); - - HTTPSampler sampler = createGetSampler("GET " + app + " metrics", amsHost, amsPort, amsPath, null, parametersMap); - - if (numLoops > 1) { - threadGroupHashTree.add(createConstantTimer(interval)); - } - - threadGroupHashTree.add(sampler); - } - } - - private HTTPSampler createGetSampler(String name, String domain, int port, String path, String encoding, Map<String, String> parameters) { - - HTTPSampler sampler = new HTTPSampler(); - sampler.setDomain(domain); - sampler.setPort(port); - sampler.setPath(path); - sampler.setMethod(HTTPConstants.GET); - - if (encoding != null) - sampler.setContentEncoding(encoding); - - for (Map.Entry<String, String> entry : parameters.entrySet()) { - sampler.addArgument(entry.getKey(), entry.getValue()); - } - sampler.setName(name); - return sampler; - } - - private LoopController createLoopController(String name, int numLoops, boolean continueForever) { - LoopController loopController = new LoopController(); - loopController.setLoops(numLoops); - loopController.setProperty(TestElement.TEST_CLASS, LoopController.class.getName()); - loopController.initialize(); - loopController.setContinueForever(continueForever); - loopController.setName(name); - return loopController; - } - - private ThreadGroup createThreadGroup(String name, int numThreads, int rampUp, LoopController loopController) { - ThreadGroup threadGroup = new ThreadGroup(); - threadGroup.setName(name); - threadGroup.setNumThreads(numThreads); - threadGroup.setRampUp(rampUp); - threadGroup.setSamplerController(loopController); - threadGroup.setProperty(TestElement.TEST_CLASS, ThreadGroup.class.getName()); - return threadGroup; - } - -} http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/TestRestMetricsSender.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/TestRestMetricsSender.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/TestRestMetricsSender.java deleted file mode 100644 index 4411be5..0000000 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/TestRestMetricsSender.java +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .loadsimulator.net; - -import org.junit.Test; - -import java.io.IOException; - -import static org.easymock.EasyMock.*; -import static org.junit.Assert.assertEquals; - -public class TestRestMetricsSender { - - @Test - public void testPushMetrics() throws Exception { - final UrlService svcMock = createStrictMock(UrlService.class); - final String payload = "test"; - final String expectedResponse = "mockResponse"; - - expect(svcMock.send(anyString())).andReturn(expectedResponse); - svcMock.disconnect(); - expectLastCall(); - - replay(svcMock); - - RestMetricsSender sender = new RestMetricsSender("expectedHostName") { - @Override - protected UrlService getConnectedUrlService() throws IOException { - return svcMock; - } - }; - String response = sender.pushMetrics(payload); - - verify(svcMock); - assertEquals("", expectedResponse, response); - } - - @Test - public void testPushMetricsFailed() throws Exception { - final UrlService svcMock = createStrictMock(UrlService.class); - final String payload = "test"; - final String expectedResponse = "mockResponse"; - RestMetricsSender sender = new RestMetricsSender("expectedHostName") { - @Override - protected UrlService getConnectedUrlService() throws IOException { - return svcMock; - } - }; - - expect(svcMock.send(anyString())).andThrow(new IOException()); - svcMock.disconnect(); - expectLastCall(); - - replay(svcMock); - - String response = sender.pushMetrics(payload); - - verify(svcMock); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/TestStdOutMetricsSender.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/TestStdOutMetricsSender.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/TestStdOutMetricsSender.java deleted file mode 100644 index 7e29ae3..0000000 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/TestStdOutMetricsSender.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.net; - - -import org.junit.Test; - -import java.io.ByteArrayOutputStream; -import java.io.PrintStream; - -public class TestStdOutMetricsSender { - - @Test - public void testPushMetrics() throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - PrintStream out = new PrintStream(baos); - StdOutMetricsSender sender = new StdOutMetricsSender("expectedHostName", out); - sender.pushMetrics("test"); - - System.out.println(baos.toString()); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/TestRandomMetricsProvider.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/TestRandomMetricsProvider.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/TestRandomMetricsProvider.java deleted file mode 100644 index 462aaf0..0000000 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/TestRandomMetricsProvider.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.util; - -import org.junit.Test; - -import static org.junit.Assert.assertTrue; - -public class TestRandomMetricsProvider { - - @Test - public void testReturnSingle() { - double from = 5.25; - double to = 5.40; - RandomMetricsProvider provider = new RandomMetricsProvider(from, to); - double metric = provider.next(); - - assertTrue("Generated metric should be in range", from < metric && metric < to); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/TestTimeStampProvider.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/TestTimeStampProvider.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/TestTimeStampProvider.java deleted file mode 100644 index dd513aa..0000000 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/TestTimeStampProvider.java +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.util; - -import org.junit.Test; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.assertEquals; - -public class TestTimeStampProvider { - - @Test - public void testReturnSingle() { - long startTime = 1411663170112L; - int timeStep = 5000; - TimeStampProvider tm = new TimeStampProvider(startTime, timeStep, 0); - - long tStamp = tm.next(); - - assertEquals("First generated timestamp should match starttime", startTime, tStamp); - } - - @Test - public void testReturnTstampsForSendInterval() throws Exception { - long startTime = 0; - int collectInterval = 5; - int sendInterval = 30; - TimeStampProvider tsp = new TimeStampProvider(startTime, collectInterval, sendInterval); - - long[] timestamps = tsp.timestampsForNextInterval(); - - assertThat(timestamps) - .hasSize(6) - .containsOnly(0, 5, 10, 15, 20, 25); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractMiniHBaseClusterTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractMiniHBaseClusterTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractMiniHBaseClusterTest.java deleted file mode 100644 index c4cebd6..0000000 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractMiniHBaseClusterTest.java +++ /dev/null @@ -1,295 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline; - -import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.OUT_OFF_BAND_DATA_TIME_ALLOWANCE; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_RECORD_TABLE_NAME; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.UPSERT_METRICS_SQL; -import static org.apache.phoenix.end2end.ParallelStatsDisabledIT.tearDownMiniCluster; -import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; -import static org.assertj.core.api.Assertions.assertThat; - -import java.io.IOException; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.IntegrationTestingUtility; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; -import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics; -import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils; -import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixConnectionProvider; -import org.apache.hadoop.yarn.util.timeline.TimelineUtils; -import org.apache.log4j.Level; -import org.apache.log4j.Logger; -import org.apache.phoenix.hbase.index.write.IndexWriterUtils; -import org.apache.phoenix.query.BaseTest; -import org.apache.phoenix.query.QueryServices; -import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.ReadOnlyProps; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -public abstract class AbstractMiniHBaseClusterTest extends BaseTest { - - protected static final long BATCH_SIZE = 3; - protected Connection conn; - protected PhoenixHBaseAccessor hdb; - - public final Log LOG; - - public AbstractMiniHBaseClusterTest() { - LOG = LogFactory.getLog(this.getClass()); - } - - @BeforeClass - public static void doSetup() throws Exception { - Map<String, String> props = getDefaultProps(); - props.put(IntegrationTestingUtility.IS_DISTRIBUTED_CLUSTER, "false"); - props.put(QueryServices.QUEUE_SIZE_ATTRIB, Integer.toString(5000)); - props.put(IndexWriterUtils.HTABLE_THREAD_KEY, Integer.toString(100)); - // Make a small batch size to test multiple calls to reserve sequences - props.put(QueryServices.SEQUENCE_CACHE_SIZE_ATTRIB, Long.toString(BATCH_SIZE)); - // Must update config before starting server - setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator())); - } - - @AfterClass - public static void doTeardown() throws Exception { - dropNonSystemTables(); - tearDownMiniCluster(); - } - - @Before - public void setUp() throws Exception { - Logger.getLogger("org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline").setLevel(Level.DEBUG); - hdb = createTestableHBaseAccessor(); - // inits connection, starts mini cluster - conn = getConnection(getUrl()); - - hdb.initMetricSchema(); - } - - private void deleteTableIgnoringExceptions(Statement stmt, String tableName) { - try { - stmt.execute("delete from " + tableName); - } catch (Exception e) { - LOG.warn("Exception on delete table " + tableName, e); - } - } - - @After - public void tearDown() { - Connection conn = null; - Statement stmt = null; - try { - conn = getConnection(getUrl()); - stmt = conn.createStatement(); - - deleteTableIgnoringExceptions(stmt, "METRIC_AGGREGATE"); - deleteTableIgnoringExceptions(stmt, "METRIC_AGGREGATE_MINUTE"); - deleteTableIgnoringExceptions(stmt, "METRIC_AGGREGATE_HOURLY"); - deleteTableIgnoringExceptions(stmt, "METRIC_AGGREGATE_DAILY"); - deleteTableIgnoringExceptions(stmt, "METRIC_RECORD"); - deleteTableIgnoringExceptions(stmt, "METRIC_RECORD_MINUTE"); - deleteTableIgnoringExceptions(stmt, "METRIC_RECORD_HOURLY"); - deleteTableIgnoringExceptions(stmt, "METRIC_RECORD_DAILY"); - deleteTableIgnoringExceptions(stmt, "METRICS_METADATA"); - deleteTableIgnoringExceptions(stmt, "HOSTED_APPS_METADATA"); - - conn.commit(); - } catch (Exception e) { - LOG.warn("Error on deleting HBase schema.", e); - } finally { - if (stmt != null) { - try { - stmt.close(); - } catch (SQLException e) { - // Ignore - } - } - - if (conn != null) { - try { - conn.close(); - } catch (SQLException e) { - // Ignore - } - } - } - } - - public static Map<String, String> getDefaultProps() { - Map<String, String> props = new HashMap<String, String>(); - // Must update config before starting server - props.put(QueryServices.STATS_USE_CURRENT_TIME_ATTRIB, - Boolean.FALSE.toString()); - props.put("java.security.krb5.realm", ""); - props.put("java.security.krb5.kdc", ""); - return props; - } - - protected Connection getConnection(String url) throws SQLException { - Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); - Connection conn = DriverManager.getConnection(getUrl(), props); - return conn; - } - - /** - * A canary test. Will show if the infrastructure is set-up correctly. - */ - @Test - public void testClusterOK() throws Exception { - Connection conn = getConnection(getUrl()); - conn.setAutoCommit(true); - - String sampleDDL = "CREATE TABLE TEST_METRICS " + - "(TEST_COLUMN VARCHAR " + - "CONSTRAINT pk PRIMARY KEY (TEST_COLUMN)) " + - "DATA_BLOCK_ENCODING='FAST_DIFF', IMMUTABLE_ROWS=true, " + - "TTL=86400, COMPRESSION='NONE' "; - - Statement stmt = conn.createStatement(); - stmt.executeUpdate(sampleDDL); - conn.commit(); - - ResultSet rs = stmt.executeQuery( - "SELECT COUNT(TEST_COLUMN) FROM TEST_METRICS"); - - rs.next(); - long l = rs.getLong(1); - assertThat(l).isGreaterThanOrEqualTo(0); - - stmt.execute("DROP TABLE TEST_METRICS"); - conn.close(); - } - - protected PhoenixHBaseAccessor createTestableHBaseAccessor() { - Configuration metricsConf = new Configuration(); - metricsConf.set(TimelineMetricConfiguration.HBASE_COMPRESSION_SCHEME, "NONE"); - // Unit tests insert values into the future - metricsConf.setLong(OUT_OFF_BAND_DATA_TIME_ALLOWANCE, 600000); - - return - new PhoenixHBaseAccessor(new TimelineMetricConfiguration(new Configuration(), metricsConf), - new PhoenixConnectionProvider() { - @Override - public Admin getHBaseAdmin() throws IOException { - try { - return driver.getConnectionQueryServices(null, null).getAdmin(); - } catch (SQLException e) { - LOG.error(e); - } - return null; - } - - @Override - public Connection getConnection() { - Connection connection = null; - try { - connection = DriverManager.getConnection(getUrl()); - } catch (SQLException e) { - LOG.warn("Unable to connect to HBase store using Phoenix.", e); - } - return connection; - } - - }); - } - - protected void insertMetricRecords(Connection conn, TimelineMetrics metrics, long currentTime) - throws SQLException, IOException { - - List<TimelineMetric> timelineMetrics = metrics.getMetrics(); - if (timelineMetrics == null || timelineMetrics.isEmpty()) { - LOG.debug("Empty metrics insert request."); - return; - } - - PreparedStatement metricRecordStmt = null; - - try { - metricRecordStmt = conn.prepareStatement(String.format( - UPSERT_METRICS_SQL, METRICS_RECORD_TABLE_NAME)); - - for (TimelineMetric metric : timelineMetrics) { - metricRecordStmt.clearParameters(); - - if (LOG.isTraceEnabled()) { - LOG.trace("host: " + metric.getHostName() + ", " + - "metricName = " + metric.getMetricName() + ", " + - "values: " + metric.getMetricValues()); - } - double[] aggregates = AggregatorUtils.calculateAggregates( - metric.getMetricValues()); - - metricRecordStmt.setString(1, metric.getMetricName()); - metricRecordStmt.setString(2, metric.getHostName()); - metricRecordStmt.setString(3, metric.getAppId()); - metricRecordStmt.setString(4, metric.getInstanceId()); - metricRecordStmt.setLong(5, metric.getStartTime()); - metricRecordStmt.setString(6, metric.getType()); - metricRecordStmt.setDouble(7, aggregates[0]); - metricRecordStmt.setDouble(8, aggregates[1]); - metricRecordStmt.setDouble(9, aggregates[2]); - metricRecordStmt.setLong(10, (long) aggregates[3]); - String json = TimelineUtils.dumpTimelineRecordtoJSON(metric.getMetricValues()); - metricRecordStmt.setString(11, json); - - try { - metricRecordStmt.executeUpdate(); - } catch (SQLException sql) { - LOG.error(sql); - } - } - - conn.commit(); - - } finally { - if (metricRecordStmt != null) { - try { - metricRecordStmt.close(); - } catch (SQLException e) { - // Ignore - } - } - if (conn != null) { - try { - conn.close(); - } catch (SQLException sql) { - // Ignore - } - } - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractPhoenixConnectionlessTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractPhoenixConnectionlessTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractPhoenixConnectionlessTest.java deleted file mode 100644 index 1430478..0000000 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractPhoenixConnectionlessTest.java +++ /dev/null @@ -1,111 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline; - -import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver; -import org.apache.phoenix.jdbc.PhoenixTestDriver; -import org.apache.phoenix.query.BaseTest; -import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.ReadOnlyProps; -import org.apache.phoenix.util.TestUtil; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB; -import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -public abstract class AbstractPhoenixConnectionlessTest extends BaseTest { - - protected static String getUrl() { - return TestUtil.PHOENIX_CONNECTIONLESS_JDBC_URL; - } - - protected static String getUrl(String tenantId) { - return getUrl() + ';' + TENANT_ID_ATTRIB + '=' + tenantId; - } - - protected static PhoenixTestDriver driver; - - private static void startServer(String url) throws Exception { - assertNull(driver); - // only load the test driver if we are testing locally - for integration tests, we want to - // test on a wider scale - if (PhoenixEmbeddedDriver.isTestUrl(url)) { - driver = initDriver(ReadOnlyProps.EMPTY_PROPS); - assertTrue(DriverManager.getDriver(url) == driver); - driver.connect(url, PropertiesUtil.deepCopy(TEST_PROPERTIES)); - } - } - - protected static synchronized PhoenixTestDriver initDriver(ReadOnlyProps props) throws Exception { - if (driver == null) { - driver = new PhoenixTestDriver(props); - DriverManager.registerDriver(driver); - } - return driver; - } - - private String connUrl; - - @Before - public void setup() throws Exception { - connUrl = getUrl(); - startServer(connUrl); - } - - @Test - public void testStorageSystemInitialized() throws Exception { - String sampleDDL = "CREATE TABLE TEST_METRICS (TEST_COLUMN VARCHAR " + - "CONSTRAINT pk PRIMARY KEY (TEST_COLUMN)) DATA_BLOCK_ENCODING='FAST_DIFF', " + - "IMMUTABLE_ROWS=true, TTL=86400, COMPRESSION='SNAPPY'"; - - Connection conn = null; - PreparedStatement stmt = null; - try { - conn = DriverManager.getConnection(connUrl); - stmt = conn.prepareStatement(sampleDDL); - stmt.execute(); - conn.commit(); - } finally { - if (stmt != null) { - stmt.close(); - } - if (conn != null) { - conn.close(); - } - } - } - - @After - public void tearDown() throws Exception { - if (driver != null) { - try { - driver.close(); - } finally { - PhoenixTestDriver phoenixTestDriver = driver; - driver = null; - DriverManager.deregisterDriver(phoenixTestDriver); - } - } - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/FunctionTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/FunctionTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/FunctionTest.java deleted file mode 100644 index 188f634..0000000 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/FunctionTest.java +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline; - -import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.Function; -import org.junit.Ignore; -import org.junit.Test; - -import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.Function.fromMetricName; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.Function.ReadFunction.AVG; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.Function.PostProcessingFunction.RATE; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.Function.PostProcessingFunction.DIFF; -import static org.assertj.core.api.Assertions.assertThat; - -public class FunctionTest { - - @Test - public void testCreation() throws Exception { - Function f = fromMetricName("Metric._avg"); - assertThat(f).isEqualTo(new Function(AVG, null)); - - f = fromMetricName("Metric._rate._avg"); - assertThat(f).isEqualTo(new Function(AVG, RATE)); - - f = fromMetricName("bytes_in"); - assertThat(f).isEqualTo(Function.DEFAULT_VALUE_FUNCTION); - - // Rate support without aggregates - f = fromMetricName("Metric._rate"); - assertThat(f).isEqualTo(new Function(null, RATE)); - - // Diff support - f = fromMetricName("Metric._diff._avg"); - assertThat(f).isEqualTo(new Function(AVG, DIFF)); - - // Diff support without aggregates - f = fromMetricName("Metric._diff"); - assertThat(f).isEqualTo(new Function(null, DIFF)); - - } - - @Ignore // If unknown function: behavior is best effort query without function - @Test(expected = Function.FunctionFormatException.class) - public void testNotAFunction() throws Exception { - fromMetricName("bytes._not._afunction"); - } -}
