YARN-3551. Consolidate data model change according to the backend implementation (Zhijie Shen via sale)
(cherry picked from commit 557a3950bddc837469244835f5577899080115d8) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/84147698 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/84147698 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/84147698 Branch: refs/heads/YARN-2928 Commit: 841476988c54bdb42a9837951e7cc36a2f6bb38a Parents: 8f0b1ca Author: Sangjin Lee <sj...@apache.org> Authored: Mon May 4 16:10:20 2015 -0700 Committer: Sangjin Lee <sj...@apache.org> Committed: Tue Aug 25 10:47:10 2015 -0700 ---------------------------------------------------------------------- .../mapred/TimelineServicePerformanceV2.java | 2 +- hadoop-yarn-project/CHANGES.txt | 3 + .../records/timelineservice/TimelineEntity.java | 16 +-- .../records/timelineservice/TimelineMetric.java | 131 +++++++++---------- .../TestTimelineServiceRecords.java | 81 +++++++++--- .../monitor/ContainersMonitorImpl.java | 4 +- .../TestTimelineServiceClientIntegration.java | 6 + 7 files changed, 146 insertions(+), 97 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/84147698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java index de46617..1c2e28d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java @@ -261,7 +261,7 @@ public class TimelineServicePerformanceV2 extends Configured implements Tool { // add a metric TimelineMetric metric = new TimelineMetric(); metric.setId("foo_metric"); - metric.setSingleData(123456789L); + metric.addValue(System.currentTimeMillis(), 123456789L); entity.addMetric(metric); // add a config entity.addConfig("foo", "bar"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/84147698/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 63243fb..41ed1ec 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -58,6 +58,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1 YARN-3431. Sub resources of timeline entity needs to be passed to a separate endpoint. (Zhijie Shen via junping_du) + YARN-3551. Consolidate data model change according to the backend + implementation (Zhijie Shen via sjlee) + IMPROVEMENTS OPTIMIZATIONS http://git-wip-us.apache.org/repos/asf/hadoop/blob/84147698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java index 6cab753..3be7f52 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java @@ -80,7 +80,7 @@ public class TimelineEntity { private TimelineEntity real; private Identifier identifier; private HashMap<String, Object> info = new HashMap<>(); - private HashMap<String, Object> configs = new HashMap<>(); + private HashMap<String, String> configs = new HashMap<>(); private Set<TimelineMetric> metrics = new HashSet<>(); private Set<TimelineEvent> events = new HashSet<>(); private HashMap<String, Set<String>> isRelatedToEntities = new HashMap<>(); @@ -213,7 +213,7 @@ public class TimelineEntity { // required by JAXB @InterfaceAudience.Private @XmlElement(name = "configs") - public HashMap<String, Object> getConfigsJAXB() { + public HashMap<String, String> getConfigsJAXB() { if (real == null) { return configs; } else { @@ -221,7 +221,7 @@ public class TimelineEntity { } } - public Map<String, Object> getConfigs() { + public Map<String, String> getConfigs() { if (real == null) { return configs; } else { @@ -229,19 +229,19 @@ public class TimelineEntity { } } - public void setConfigs(Map<String, Object> configs) { + public void setConfigs(Map<String, String> configs) { if (real == null) { if (configs != null && !(configs instanceof HashMap)) { - this.configs = new HashMap<String, Object>(configs); + this.configs = new HashMap<String, String>(configs); } else { - this.configs = (HashMap<String, Object>) configs; + this.configs = (HashMap<String, String>) configs; } } else { real.setConfigs(configs); } } - public void addConfigs(Map<String, Object> configs) { + public void addConfigs(Map<String, String> configs) { if (real == null) { this.configs.putAll(configs); } else { @@ -249,7 +249,7 @@ public class TimelineEntity { } } - public void addConfig(String key, Object value) { + public void addConfig(String key, String value) { if (real == null) { configs.put(key, value); } else { http://git-wip-us.apache.org/repos/asf/hadoop/blob/84147698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java index 57babbf..c897d39 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java @@ -24,113 +24,104 @@ import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; -import java.util.HashMap; -import java.util.LinkedHashMap; +import java.util.Comparator; import java.util.Map; +import java.util.TreeMap; @XmlRootElement(name = "metric") @XmlAccessorType(XmlAccessType.NONE) @InterfaceAudience.Public @InterfaceStability.Unstable public class TimelineMetric { - private String id; - private HashMap<String, Object> info = new HashMap<>(); - private Object singleData; - private HashMap<Long, Object> timeSeries = new LinkedHashMap<>(); - private long startTime; - private long endTime; - - public TimelineMetric() { + public static enum Type { + SINGLE_VALUE, + TIME_SERIES } - @XmlElement(name = "id") - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } + private Type type; + private String id; + private Comparator<Long> reverseComparator = new Comparator<Long>() { + @Override + public int compare(Long l1, Long l2) { + return -l1.compareTo(l2); + } + }; + private TreeMap<Long, Number> values = new TreeMap<>(reverseComparator); - // required by JAXB - @InterfaceAudience.Private - @XmlElement(name = "info") - public HashMap<String, Object> getInfoJAXB() { - return info; + public TimelineMetric() { + this(Type.SINGLE_VALUE); } - public Map<String, Object> getInfo() { - return info; + public TimelineMetric(Type type) { + this.type = type; } - public void setInfo(Map<String, Object> info) { - if (info != null && !(info instanceof HashMap)) { - this.info = new HashMap<String, Object>(info); - } else { - this.info = (HashMap<String, Object>) info; - } - } - public void addInfo(Map<String, Object> info) { - this.info.putAll(info); + @XmlElement(name = "type") + public Type getType() { + return type; } - public void addInfo(String key, Object value) { - info.put(key, value); + public void setType(Type type) { + this.type = type; } - @XmlElement(name = "data") - public Object getSingleData() { - return singleData; + @XmlElement(name = "id") + public String getId() { + return id; } - public void setSingleData(Object singleData) { - this.singleData = singleData; + public void setId(String id) { + this.id = id; } // required by JAXB @InterfaceAudience.Private - @XmlElement(name = "timeseries") - public HashMap<Long, Object> getTimeSeriesJAXB() { - return timeSeries; + @XmlElement(name = "values") + public TreeMap<Long, Number> getValuesJAXB() { + return values; } - public Map<Long, Object> getTimeSeries() { - return timeSeries; + public Map<Long, Number> getValues() { + return values; } - public void setTimeSeries(Map<Long, Object> timeSeries) { - if (timeSeries != null && !(timeSeries instanceof LinkedHashMap)) { - this.timeSeries = new LinkedHashMap<Long, Object>(timeSeries); + public void setValues(Map<Long, Number> values) { + if (type == Type.SINGLE_VALUE) { + overwrite(values); } else { - this.timeSeries = (LinkedHashMap<Long, Object>) timeSeries; + if (values != null) { + this.values = new TreeMap<Long, Number>(reverseComparator); + this.values.putAll(values); + } else { + this.values = null; + } } } - public void addTimeSeries(Map<Long, Object> timeSeries) { - this.timeSeries.putAll(timeSeries); - } - - public void addTimeSeriesData(long timestamp, Object value) { - timeSeries.put(timestamp, value); - } - - @XmlElement(name = "starttime") - public long getStartTime() { - return startTime; - } - - public void setStartTime(long startTime) { - this.startTime = startTime; + public void addValues(Map<Long, Number> values) { + if (type == Type.SINGLE_VALUE) { + overwrite(values); + } else { + this.values.putAll(values); + } } - @XmlElement(name = "endtime") - public long getEndTime() { - return endTime; + public void addValue(long timestamp, Number value) { + if (type == Type.SINGLE_VALUE) { + values.clear(); + } + values.put(timestamp, value); } - public void setEndTime(long endTime) { - this.endTime = endTime; + private void overwrite(Map<Long, Number> values) { + if (values.size() > 1) { + throw new IllegalArgumentException( + "Values cannot contain more than one point in " + + Type.SINGLE_VALUE + " mode"); + } + this.values.clear(); + this.values.putAll(values); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/84147698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java index caa3f3f..95b922e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java @@ -28,7 +28,11 @@ import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import org.junit.Test; import org.junit.Assert; +import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; public class TestTimelineServiceRecords { @@ -41,38 +45,83 @@ public class TestTimelineServiceRecords { entity.setType("test type 1"); entity.setId("test id 1"); entity.addInfo("test info key 1", "test info value 1"); - entity.addInfo("test info key 2", "test info value 2"); + entity.addInfo("test info key 2", + Arrays.asList("test info value 2", "test info value 3")); + entity.addInfo("test info key 3", true); + Assert.assertTrue( + entity.getInfo().get("test info key 3") instanceof Boolean); entity.addConfig("test config key 1", "test config value 1"); entity.addConfig("test config key 2", "test config value 2"); - TimelineMetric metric1 = new TimelineMetric(); + + TimelineMetric metric1 = + new TimelineMetric(TimelineMetric.Type.TIME_SERIES); metric1.setId("test metric id 1"); - metric1.addInfo("test info key 1", "test info value 1"); - metric1.addInfo("test info key 2", "test info value 2"); - metric1.addTimeSeriesData(1L, "test time series 1"); - metric1.addTimeSeriesData(2L, "test time series 2"); - metric1.setStartTime(0L); - metric1.setEndTime(1L); + metric1.addValue(1L, 1.0F); + metric1.addValue(3L, 3.0D); + metric1.addValue(2L, 2); + Assert.assertEquals(TimelineMetric.Type.TIME_SERIES, metric1.getType()); + Iterator<Map.Entry<Long, Number>> itr = + metric1.getValues().entrySet().iterator(); + Map.Entry<Long, Number> entry = itr.next(); + Assert.assertEquals(new Long(3L), entry.getKey()); + Assert.assertEquals(new Double(3.0D), entry.getValue()); + entry = itr.next(); + Assert.assertEquals(new Long(2L), entry.getKey()); + Assert.assertEquals(new Integer(2), entry.getValue()); + entry = itr.next(); + Assert.assertEquals(new Long(1L), entry.getKey()); + Assert.assertEquals(new Float(1.0F), entry.getValue()); + Assert.assertFalse(itr.hasNext()); entity.addMetric(metric1); - TimelineMetric metric2 = new TimelineMetric(); + + TimelineMetric metric2 = + new TimelineMetric(TimelineMetric.Type.SINGLE_VALUE); metric2.setId("test metric id 1"); - metric2.addInfo("test info key 1", "test info value 1"); - metric2.addInfo("test info key 2", "test info value 2"); - metric2.setSingleData("test info value 3"); - metric1.setStartTime(0L); - metric1.setEndTime(1L); + metric2.addValue(3L, (short) 3); + Assert.assertEquals(TimelineMetric.Type.SINGLE_VALUE, metric2.getType()); + Assert.assertTrue( + metric2.getValues().values().iterator().next() instanceof Short); + Map<Long, Number> points = new HashMap<>(); + points.put(4L, 4.0D); + points.put(5L, 5.0D); + try { + metric2.setValues(points); + Assert.fail(); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().contains( + "Values cannot contain more than one point in")); + } + try { + metric2.addValues(points); + Assert.fail(); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().contains( + "Values cannot contain more than one point in")); + } entity.addMetric(metric2); + TimelineEvent event1 = new TimelineEvent(); event1.setId("test event id 1"); event1.addInfo("test info key 1", "test info value 1"); - event1.addInfo("test info key 2", "test info value 2"); + event1.addInfo("test info key 2", + Arrays.asList("test info value 2", "test info value 3")); + event1.addInfo("test info key 3", true); + Assert.assertTrue( + event1.getInfo().get("test info key 3") instanceof Boolean); event1.setTimestamp(0L); entity.addEvent(event1); + TimelineEvent event2 = new TimelineEvent(); event2.setId("test event id 2"); event2.addInfo("test info key 1", "test info value 1"); - event2.addInfo("test info key 2", "test info value 2"); + event2.addInfo("test info key 2", + Arrays.asList("test info value 2", "test info value 3")); + event2.addInfo("test info key 3", true); + Assert.assertTrue( + event2.getInfo().get("test info key 3") instanceof Boolean); event2.setTimestamp(1L); entity.addEvent(event2); + entity.setCreatedTime(0L); entity.setModifiedTime(1L); entity.addRelatesToEntity("test type 2", "test id 2"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/84147698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java index a1707b7..68748f0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java @@ -566,7 +566,7 @@ public class ContainersMonitorImpl extends AbstractService implements ResourceCalculatorProcessTree.UNAVAILABLE) { TimelineMetric memoryMetric = new TimelineMetric(); memoryMetric.setId(ContainerMetric.MEMORY.toString() + pId); - memoryMetric.addTimeSeriesData(currentTime, currentPmemUsage); + memoryMetric.addValue(currentTime, currentPmemUsage); entity.addMetric(memoryMetric); } // if cpuUsageTotalCoresPercentage data is available @@ -574,7 +574,7 @@ public class ContainersMonitorImpl extends AbstractService implements ResourceCalculatorProcessTree.UNAVAILABLE) { TimelineMetric cpuMetric = new TimelineMetric(); cpuMetric.setId(ContainerMetric.CPU.toString() + pId); - cpuMetric.addTimeSeriesData(currentTime, + cpuMetric.addValue(currentTime, cpuUsageTotalCoresPercentage); entity.addMetric(cpuMetric); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/84147698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java index 044148a..69031a2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java @@ -81,6 +81,12 @@ public class TestTimelineServiceClientIntegration { TimelineEntity entity = new TimelineEntity(); entity.setType("test entity type"); entity.setId("test entity id"); + TimelineMetric metric = + new TimelineMetric(TimelineMetric.Type.TIME_SERIES); + metric.setId("test metric id"); + metric.addValue(1L, 1.0D); + metric.addValue(2L, 2.0D); + entity.addMetric(metric); client.putEntities(entity); client.putEntitiesAsync(entity); } finally {
