YARN-3431. Sub resources of timeline entity needs to be passed to a separate endpoint. Contributed By Zhijie Shen.
(cherry picked from commit fa5cc75245a6dba549620a8b26c7b4a8aed9838e) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f0bc3390 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f0bc3390 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f0bc3390 Branch: refs/heads/YARN-2928 Commit: f0bc3390f9eacda95104938b17b757c730182426 Parents: 63c6606 Author: Junping Du <[email protected]> Authored: Mon Apr 27 11:28:32 2015 -0700 Committer: Vinod Kumar Vavilapalli <[email protected]> Committed: Fri Aug 14 11:23:23 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 + .../ApplicationAttemptEntity.java | 13 +- .../timelineservice/ApplicationEntity.java | 22 +- .../records/timelineservice/ClusterEntity.java | 12 +- .../timelineservice/ContainerEntity.java | 13 +- .../api/records/timelineservice/FlowEntity.java | 80 +++-- .../HierarchicalTimelineEntity.java | 124 +++---- .../records/timelineservice/QueueEntity.java | 36 +++ .../records/timelineservice/TimelineEntity.java | 322 +++++++++++++++---- .../records/timelineservice/TimelineQueue.java | 35 -- .../records/timelineservice/TimelineUser.java | 35 -- .../api/records/timelineservice/UserEntity.java | 36 +++ .../TestTimelineServiceRecords.java | 91 ++++-- .../TestTimelineServiceClientIntegration.java | 44 ++- .../collector/TimelineCollectorWebService.java | 65 +++- 15 files changed, 654 insertions(+), 277 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0bc3390/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 0a8fc6e..2fe104e 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -55,6 +55,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1 YARN-3390. Reuse TimelineCollectorManager for RM (Zhijie Shen via sjlee) + YARN-3431. Sub resources of timeline entity needs to be passed to a separate + endpoint. (Zhijie Shen via junping_du) + IMPROVEMENTS OPTIMIZATIONS http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0bc3390/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationAttemptEntity.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationAttemptEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationAttemptEntity.java index 9dc0c1d..734c741 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationAttemptEntity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationAttemptEntity.java @@ -20,16 +20,17 @@ package org.apache.hadoop.yarn.api.records.timelineservice; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlRootElement; - -@XmlRootElement(name = "appattempt") -@XmlAccessorType(XmlAccessType.NONE) @InterfaceAudience.Public @InterfaceStability.Unstable public class ApplicationAttemptEntity extends HierarchicalTimelineEntity { public ApplicationAttemptEntity() { super(TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString()); } + + public ApplicationAttemptEntity(TimelineEntity entity) { + super(entity); + if (!entity.getType().equals(TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString())) { + throw new IllegalArgumentException("Incompatible entity type: " + getId()); + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0bc3390/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java index 45ec520..183d8d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java @@ -20,28 +20,28 @@ package org.apache.hadoop.yarn.api.records.timelineservice; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlRootElement; - -@XmlRootElement(name = "application") -@XmlAccessorType(XmlAccessType.NONE) @InterfaceAudience.Public @InterfaceStability.Unstable public class ApplicationEntity extends HierarchicalTimelineEntity { - private String queue; + public static final String QUEUE_INFO_KEY = + TimelineEntity.SYSTEM_INFO_KEY_PREFIX + "QUEUE"; public ApplicationEntity() { super(TimelineEntityType.YARN_APPLICATION.toString()); } - @XmlElement(name = "queue") + public ApplicationEntity(TimelineEntity entity) { + super(entity); + if (!entity.getType().equals(TimelineEntityType.YARN_APPLICATION.toString())) { + throw new IllegalArgumentException("Incompatible entity type: " + getId()); + } + } + public String getQueue() { - return queue; + return getInfo().get(QUEUE_INFO_KEY).toString(); } public void setQueue(String queue) { - this.queue = queue; + addInfo(QUEUE_INFO_KEY, queue); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0bc3390/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ClusterEntity.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ClusterEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ClusterEntity.java index a4278c0..94eefa8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ClusterEntity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ClusterEntity.java @@ -20,12 +20,6 @@ package org.apache.hadoop.yarn.api.records.timelineservice; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlRootElement; - -@XmlRootElement(name = "cluster") -@XmlAccessorType(XmlAccessType.NONE) @InterfaceAudience.Public @InterfaceStability.Unstable public class ClusterEntity extends HierarchicalTimelineEntity { @@ -33,4 +27,10 @@ public class ClusterEntity extends HierarchicalTimelineEntity { super(TimelineEntityType.YARN_CLUSTER.toString()); } + public ClusterEntity(TimelineEntity entity) { + super(entity); + if (!entity.getType().equals(TimelineEntityType.YARN_CLUSTER.toString())) { + throw new IllegalArgumentException("Incompatible entity type: " + getId()); + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0bc3390/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ContainerEntity.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ContainerEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ContainerEntity.java index cde6040..c7d6bce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ContainerEntity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ContainerEntity.java @@ -20,16 +20,17 @@ package org.apache.hadoop.yarn.api.records.timelineservice; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlRootElement; - -@XmlRootElement(name = "container") -@XmlAccessorType(XmlAccessType.NONE) @InterfaceAudience.Public @InterfaceStability.Unstable public class ContainerEntity extends HierarchicalTimelineEntity { public ContainerEntity() { super(TimelineEntityType.YARN_CONTAINER.toString()); } + + public ContainerEntity(TimelineEntity entity) { + super(entity); + if (!entity.getType().equals(TimelineEntityType.YARN_CONTAINER.toString())) { + throw new IllegalArgumentException("Incompatible entity type: " + getId()); + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0bc3390/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowEntity.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowEntity.java index 0765f00..4554778 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowEntity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowEntity.java @@ -20,62 +20,84 @@ package org.apache.hadoop.yarn.api.records.timelineservice; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlRootElement; -@XmlRootElement(name = "flow") -@XmlAccessorType(XmlAccessType.NONE) @InterfaceAudience.Public @InterfaceStability.Unstable public class FlowEntity extends HierarchicalTimelineEntity { - private String user; - private String version; - private String run; + public static final String USER_INFO_KEY = + TimelineEntity.SYSTEM_INFO_KEY_PREFIX + "USER"; + public static final String FLOW_NAME_INFO_KEY = + TimelineEntity.SYSTEM_INFO_KEY_PREFIX + "FLOW_NAME"; + public static final String FLOW_VERSION_INFO_KEY = + TimelineEntity.SYSTEM_INFO_KEY_PREFIX + "FLOW_VERSION"; + public static final String FLOW_RUN_ID_INFO_KEY = + TimelineEntity.SYSTEM_INFO_KEY_PREFIX + "FLOW_RUN_ID"; public FlowEntity() { super(TimelineEntityType.YARN_FLOW.toString()); } + public FlowEntity(TimelineEntity entity) { + super(entity); + if (!entity.getType().equals(TimelineEntityType.YARN_FLOW.toString())) { + throw new IllegalArgumentException("Incompatible entity type: " + getId()); + } + } + + @XmlElement(name = "id") @Override public String getId() { - //Flow id schema: user@flow_name(or id)/version/run - StringBuilder sb = new StringBuilder(); - sb.append(user); - sb.append('@'); - sb.append(super.getId()); - sb.append('/'); - sb.append(version); - sb.append('/'); - sb.append(run); - return sb.toString(); + //Flow id schema: user@flow_name(or id)/version/run_id + String id = super.getId(); + if (id == null) { + StringBuilder sb = new StringBuilder(); + sb.append(getInfo().get(USER_INFO_KEY).toString()); + sb.append('@'); + sb.append(getInfo().get(FLOW_NAME_INFO_KEY).toString()); + sb.append('/'); + sb.append(getInfo().get(FLOW_VERSION_INFO_KEY).toString()); + sb.append('/'); + sb.append(getInfo().get(FLOW_RUN_ID_INFO_KEY).toString()); + id = sb.toString(); + setId(id); + } + return id; } - @XmlElement(name = "user") public String getUser() { - return user; + Object user = getInfo().get(USER_INFO_KEY); + return user == null ? null : user.toString(); } public void setUser(String user) { - this.user = user; + addInfo(USER_INFO_KEY, user); + } + + public String getName() { + Object name = getInfo().get(FLOW_NAME_INFO_KEY); + return name == null ? null : name.toString(); + } + + public void setName(String name) { + addInfo(FLOW_NAME_INFO_KEY, name); } - @XmlElement(name = "version") public String getVersion() { - return version; + Object version = getInfo().get(FLOW_VERSION_INFO_KEY); + return version == null ? null : version.toString(); } public void setVersion(String version) { - this.version = version; + addInfo(FLOW_VERSION_INFO_KEY, version); } - @XmlElement(name = "run") - public String getRun() { - return run; + public long getRunId() { + Object runId = getInfo().get(FLOW_RUN_ID_INFO_KEY); + return runId == null ? 0L : (Long) runId; } - public void setRun(String run) { - this.run = run; + public void setRunId(long runId) { + addInfo(FLOW_RUN_ID_INFO_KEY, runId); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0bc3390/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/HierarchicalTimelineEntity.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/HierarchicalTimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/HierarchicalTimelineEntity.java index 49576de..6235da0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/HierarchicalTimelineEntity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/HierarchicalTimelineEntity.java @@ -17,93 +17,98 @@ */ package org.apache.hadoop.yarn.api.records.timelineservice; +import com.google.common.base.Joiner; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlElement; -import java.util.HashMap; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; -import java.util.Map; +import java.util.List; import java.util.Set; -@XmlAccessorType(XmlAccessType.NONE) @InterfaceAudience.Public @InterfaceStability.Unstable public abstract class HierarchicalTimelineEntity extends TimelineEntity { - private Identifier parent; - private HashMap<String, Set<String>> children = new HashMap<>(); + public static final String PARENT_INFO_KEY = + TimelineEntity.SYSTEM_INFO_KEY_PREFIX + "PARENT_ENTITY"; + public static final String CHILDREN_INFO_KEY = + TimelineEntity.SYSTEM_INFO_KEY_PREFIX + "CHILDREN_ENTITY"; + + HierarchicalTimelineEntity(TimelineEntity entity) { + super(entity); + } HierarchicalTimelineEntity(String type) { super(type); } - @XmlElement(name = "parent") public Identifier getParent() { - return parent; + Object obj = getInfo().get(PARENT_INFO_KEY); + if (obj != null) { + if (obj instanceof Identifier) { + return (Identifier) obj; + } else { + throw new YarnRuntimeException( + "Parent info is invalid identifier object"); + } + } + return null; } public void setParent(Identifier parent) { validateParent(parent.getType()); - this.parent = parent; + addInfo(PARENT_INFO_KEY, parent); } public void setParent(String type, String id) { - validateParent(type); - parent = new Identifier(); - parent.setType(type); - parent.setId(id); + setParent(new Identifier(type, id)); } - // required by JAXB - @InterfaceAudience.Private - // comment out XmlElement here because it cause UnrecognizedPropertyException - // TODO we need a better fix - //@XmlElement(name = "children") - public HashMap<String, Set<String>> getChildrenJAXB() { + public Set<Identifier> getChildren() { + Object identifiers = getInfo().get(CHILDREN_INFO_KEY); + if (identifiers == null) { + return new HashSet<>(); + } + TimelineEntityType thisType = TimelineEntityType.valueOf(getType()); + if (identifiers instanceof Set<?>) { + for (Object identifier : (Set<?>) identifiers) { + if (!(identifier instanceof Identifier)) { + throw new YarnRuntimeException( + "Children info contains invalid identifier object"); + } else { + validateChild((Identifier) identifier, thisType); + } + } + } else { + throw new YarnRuntimeException( + "Children info is invalid identifier set"); + } + Set<Identifier> children = (Set<Identifier>) identifiers; return children; } - public Map<String, Set<String>> getChildren() { - return children; + public void setChildren(Set<Identifier> children) { + addInfo(CHILDREN_INFO_KEY, children); } - public void setChildren(Map<String, Set<String>> children) { - validateChildren(children); - if (children != null && !(children instanceof HashMap)) { - this.children = new HashMap<String, Set<String>>(children); - } else { - this.children = (HashMap) children; + public void addChildren(Set<Identifier> children) { + TimelineEntityType thisType = TimelineEntityType.valueOf(getType()); + for (Identifier child : children) { + validateChild(child, thisType); } + Set<Identifier> existingChildren = getChildren(); + existingChildren.addAll(children); + setChildren(existingChildren); } - public void addChildren(Map<String, Set<String>> children) { - validateChildren(children); - for (Map.Entry<String, Set<String>> entry : children.entrySet()) { - Set<String> ids = this.children.get(entry.getKey()); - if (ids == null) { - ids = new HashSet<>(); - this.children.put(entry.getKey(), ids); - } - ids.addAll(entry.getValue()); - } + public void addChild(Identifier child) { + addChildren(Collections.singleton(child)); } public void addChild(String type, String id) { - TimelineEntityType thisType = TimelineEntityType.valueOf(getType()); - TimelineEntityType childType = TimelineEntityType.valueOf(type); - if (thisType.isChild(childType)) { - Set<String> ids = children.get(type); - if (ids == null) { - ids = new HashSet<>(); - children.put(type, ids); - } - ids.add(id); - } else { - throw new IllegalArgumentException( - type + " is not the acceptable child of " + this.getType()); - } + addChild(new Identifier(type, id)); } private void validateParent(String type) { @@ -115,15 +120,12 @@ public abstract class HierarchicalTimelineEntity extends TimelineEntity { } } - private void validateChildren(Map<String, Set<String>> children) { - TimelineEntityType thisType = TimelineEntityType.valueOf(getType()); - for (Map.Entry<String, Set<String>> entry : children.entrySet()) { - TimelineEntityType childType = TimelineEntityType.valueOf(entry.getKey()); - if (!thisType.isChild(childType)) { - throw new IllegalArgumentException( - entry.getKey() + " is not the acceptable child of " + - this.getType()); - } + private void validateChild(Identifier child, TimelineEntityType thisType) { + TimelineEntityType childType = TimelineEntityType.valueOf(child.getType()); + if (!thisType.isChild(childType)) { + throw new IllegalArgumentException( + child.getType() + " is not the acceptable child of " + + this.getType()); } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0bc3390/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/QueueEntity.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/QueueEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/QueueEntity.java new file mode 100644 index 0000000..d1a1f19 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/QueueEntity.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.records.timelineservice; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + [email protected] [email protected] +public class QueueEntity extends HierarchicalTimelineEntity { + public QueueEntity() { + super(TimelineEntityType.YARN_QUEUE.toString()); + } + + public QueueEntity(TimelineEntity entity) { + super(entity); + if (!entity.getType().equals(TimelineEntityType.YARN_QUEUE.toString())) { + throw new IllegalArgumentException("Incompatible entity type: " + getId()); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0bc3390/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java index 1afb564..6cab753 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java @@ -34,6 +34,7 @@ import java.util.Set; @InterfaceAudience.Public @InterfaceStability.Unstable public class TimelineEntity { + protected final static String SYSTEM_INFO_KEY_PREFIX = "SYSTEM_INFO_"; @XmlRootElement(name = "identifier") @XmlAccessorType(XmlAccessType.NONE) @@ -41,6 +42,11 @@ public class TimelineEntity { private String type; private String id; + public Identifier(String type, String id) { + this.type = type; + this.id = id; + } + public Identifier() { } @@ -62,8 +68,16 @@ public class TimelineEntity { public void setId(String id) { this.id = id; } + + @Override + public String toString() { + return "TimelineEntity[" + + "type='" + type + '\'' + + ", id='" + id + '\'' + "]"; + } } + private TimelineEntity real; private Identifier identifier; private HashMap<String, Object> info = new HashMap<>(); private HashMap<String, Object> configs = new HashMap<>(); @@ -78,6 +92,22 @@ public class TimelineEntity { identifier = new Identifier(); } + /** + * <p> + * The constuctor is used to construct a proxy {@link TimelineEntity} or its + * subclass object from the real entity object that carries information. + * </p> + * + * <p> + * It is usually used in the case where we want to recover class polymorphism + * after deserializing the entity from its JSON form. + * </p> + * @param entity the real entity that carries information + */ + public TimelineEntity(TimelineEntity entity) { + real = entity.getReal(); + } + protected TimelineEntity(String type) { this(); identifier.type = type; @@ -85,216 +115,378 @@ public class TimelineEntity { @XmlElement(name = "type") public String getType() { - return identifier.type; + if (real == null) { + return identifier.type; + } else { + return real.getType(); + } } public void setType(String type) { - identifier.type = type; + if (real == null) { + identifier.type = type; + } else { + real.setType(type); + } } @XmlElement(name = "id") public String getId() { - return identifier.id; + if (real == null) { + return identifier.id; + } else { + return real.getId(); + } } public void setId(String id) { - identifier.id = id; + if (real == null) { + identifier.id = id; + } else { + real.setId(id); + } } public Identifier getIdentifier() { - return identifier; + if (real == null) { + return identifier; + } else { + return real.getIdentifier(); + } } public void setIdentifier(Identifier identifier) { - this.identifier = identifier; + if (real == null) { + this.identifier = identifier; + } else { + real.setIdentifier(identifier); + } } // required by JAXB @InterfaceAudience.Private @XmlElement(name = "info") public HashMap<String, Object> getInfoJAXB() { - return info; + if (real == null) { + return info; + } else { + return real.getInfoJAXB(); + } } public Map<String, Object> getInfo() { - return info; + if (real == null) { + return info; + } else { + return real.getInfo(); + } } public void setInfo(Map<String, Object> info) { - if (info != null && !(info instanceof HashMap)) { - this.info = new HashMap<String, Object>(info); + if (real == null) { + if (info != null && !(info instanceof HashMap)) { + this.info = new HashMap<String, Object>(info); + } else { + this.info = (HashMap<String, Object>) info; + } } else { - this.info = (HashMap<String, Object>) info; + real.setInfo(info); } } public void addInfo(Map<String, Object> info) { - this.info.putAll(info); + if (real == null) { + this.info.putAll(info); + } else { + real.addInfo(info); + } } public void addInfo(String key, Object value) { - info.put(key, value); + if (real == null) { + info.put(key, value); + } else { + real.addInfo(key, value); + } } // required by JAXB @InterfaceAudience.Private @XmlElement(name = "configs") public HashMap<String, Object> getConfigsJAXB() { - return configs; + if (real == null) { + return configs; + } else { + return real.getConfigsJAXB(); + } } public Map<String, Object> getConfigs() { - return configs; + if (real == null) { + return configs; + } else { + return real.getConfigs(); + } } public void setConfigs(Map<String, Object> configs) { - if (configs != null && !(configs instanceof HashMap)) { - this.configs = new HashMap<String, Object>(configs); + if (real == null) { + if (configs != null && !(configs instanceof HashMap)) { + this.configs = new HashMap<String, Object>(configs); + } else { + this.configs = (HashMap<String, Object>) configs; + } } else { - this.configs = (HashMap<String, Object>) configs; + real.setConfigs(configs); } } public void addConfigs(Map<String, Object> configs) { - this.configs.putAll(configs); + if (real == null) { + this.configs.putAll(configs); + } else { + real.addConfigs(configs); + } } public void addConfig(String key, Object value) { - configs.put(key, value); + if (real == null) { + configs.put(key, value); + } else { + real.addConfig(key, value); + } } @XmlElement(name = "metrics") public Set<TimelineMetric> getMetrics() { - return metrics; + if (real == null) { + return metrics; + } else { + return real.getMetrics(); + } } public void setMetrics(Set<TimelineMetric> metrics) { - this.metrics = metrics; + if (real == null) { + this.metrics = metrics; + } else { + real.setMetrics(metrics); + } } public void addMetrics(Set<TimelineMetric> metrics) { - this.metrics.addAll(metrics); + if (real == null) { + this.metrics.addAll(metrics); + } else { + real.addMetrics(metrics); + } } public void addMetric(TimelineMetric metric) { - metrics.add(metric); + if (real == null) { + metrics.add(metric); + } else { + real.addMetric(metric); + } } @XmlElement(name = "events") public Set<TimelineEvent> getEvents() { - return events; + if (real == null) { + return events; + } else { + return real.getEvents(); + } } public void setEvents(Set<TimelineEvent> events) { - this.events = events; + if (real == null) { + this.events = events; + } else { + real.setEvents(events); + } } public void addEvents(Set<TimelineEvent> events) { - this.events.addAll(events); + if (real == null) { + this.events.addAll(events); + } else { + real.addEvents(events); + } } public void addEvent(TimelineEvent event) { - events.add(event); + if (real == null) { + events.add(event); + } else { + real.addEvent(event); + } } public Map<String, Set<String>> getIsRelatedToEntities() { - return isRelatedToEntities; + if (real == null) { + return isRelatedToEntities; + } else { + return real.getIsRelatedToEntities(); + } } // required by JAXB @InterfaceAudience.Private @XmlElement(name = "isrelatedto") public HashMap<String, Set<String>> getIsRelatedToEntitiesJAXB() { - return isRelatedToEntities; + if (real == null) { + return isRelatedToEntities; + } else { + return real.getIsRelatedToEntitiesJAXB(); + } } public void setIsRelatedToEntities( Map<String, Set<String>> isRelatedToEntities) { - if (isRelatedToEntities != null && !(isRelatedToEntities instanceof HashMap)) { - this.isRelatedToEntities = new HashMap<String, Set<String>>(isRelatedToEntities); + if (real == null) { + if (isRelatedToEntities != null && + !(isRelatedToEntities instanceof HashMap)) { + this.isRelatedToEntities = + new HashMap<String, Set<String>>(isRelatedToEntities); + } else { + this.isRelatedToEntities = + (HashMap<String, Set<String>>) isRelatedToEntities; + } } else { - this.isRelatedToEntities = (HashMap<String, Set<String>>) isRelatedToEntities; + real.setIsRelatedToEntities(isRelatedToEntities); } } public void addIsRelatedToEntities( Map<String, Set<String>> isRelatedToEntities) { - for (Map.Entry<String, Set<String>> entry : isRelatedToEntities - .entrySet()) { - Set<String> ids = this.isRelatedToEntities.get(entry.getKey()); - if (ids == null) { - ids = new HashSet<>(); - this.isRelatedToEntities.put(entry.getKey(), ids); + if (real == null) { + for (Map.Entry<String, Set<String>> entry : isRelatedToEntities + .entrySet()) { + Set<String> ids = this.isRelatedToEntities.get(entry.getKey()); + if (ids == null) { + ids = new HashSet<>(); + this.isRelatedToEntities.put(entry.getKey(), ids); + } + ids.addAll(entry.getValue()); } - ids.addAll(entry.getValue()); + } else { + real.addIsRelatedToEntities(isRelatedToEntities); } } public void addIsRelatedToEntity(String type, String id) { - Set<String> ids = isRelatedToEntities.get(type); - if (ids == null) { - ids = new HashSet<>(); - isRelatedToEntities.put(type, ids); + if (real == null) { + Set<String> ids = isRelatedToEntities.get(type); + if (ids == null) { + ids = new HashSet<>(); + isRelatedToEntities.put(type, ids); + } + ids.add(id); + } else { + real.addIsRelatedToEntity(type, id); } - ids.add(id); } // required by JAXB @InterfaceAudience.Private @XmlElement(name = "relatesto") public HashMap<String, Set<String>> getRelatesToEntitiesJAXB() { - return relatesToEntities; + if (real == null) { + return relatesToEntities; + } else { + return real.getRelatesToEntitiesJAXB(); + } } public Map<String, Set<String>> getRelatesToEntities() { - return relatesToEntities; + if (real == null) { + return relatesToEntities; + } else { + return real.getRelatesToEntities(); + } } public void addRelatesToEntities(Map<String, Set<String>> relatesToEntities) { - for (Map.Entry<String, Set<String>> entry : relatesToEntities.entrySet()) { - Set<String> ids = this.relatesToEntities.get(entry.getKey()); - if (ids == null) { - ids = new HashSet<>(); - this.relatesToEntities.put(entry.getKey(), ids); + if (real == null) { + for (Map.Entry<String, Set<String>> entry : relatesToEntities + .entrySet()) { + Set<String> ids = this.relatesToEntities.get(entry.getKey()); + if (ids == null) { + ids = new HashSet<>(); + this.relatesToEntities.put(entry.getKey(), ids); + } + ids.addAll(entry.getValue()); } - ids.addAll(entry.getValue()); + } else { + real.addRelatesToEntities(relatesToEntities); } } public void addRelatesToEntity(String type, String id) { - Set<String> ids = relatesToEntities.get(type); - if (ids == null) { - ids = new HashSet<>(); - relatesToEntities.put(type, ids); + if (real == null) { + Set<String> ids = relatesToEntities.get(type); + if (ids == null) { + ids = new HashSet<>(); + relatesToEntities.put(type, ids); + } + ids.add(id); + } else { + real.addRelatesToEntity(type, id); } - ids.add(id); } public void setRelatesToEntities(Map<String, Set<String>> relatesToEntities) { - if (relatesToEntities != null && !(relatesToEntities instanceof HashMap)) { - this.relatesToEntities = new HashMap<String, Set<String>>(relatesToEntities); + if (real == null) { + if (relatesToEntities != null && + !(relatesToEntities instanceof HashMap)) { + this.relatesToEntities = + new HashMap<String, Set<String>>(relatesToEntities); + } else { + this.relatesToEntities = + (HashMap<String, Set<String>>) relatesToEntities; + } } else { - this.relatesToEntities = (HashMap<String, Set<String>>) relatesToEntities; + real.setRelatesToEntities(relatesToEntities); } } @XmlElement(name = "createdtime") public long getCreatedTime() { - return createdTime; + if (real == null) { + return createdTime; + } else { + return real.getCreatedTime(); + } } public void setCreatedTime(long createdTime) { - this.createdTime = createdTime; + if (real == null) { + this.createdTime = createdTime; + } else { + real.setCreatedTime(createdTime); + } } @XmlElement(name = "modifiedtime") public long getModifiedTime() { - return modifiedTime; + if (real == null) { + return modifiedTime; + } else { + return real.getModifiedTime(); + } } public void setModifiedTime(long modifiedTime) { - this.modifiedTime = modifiedTime; + if (real == null) { + this.modifiedTime = modifiedTime; + } else { + real.setModifiedTime(modifiedTime); + } } + protected TimelineEntity getReal() { + return real == null ? this : real; + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0bc3390/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineQueue.java deleted file mode 100644 index c4039f8..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineQueue.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.api.records.timelineservice; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlRootElement; - -@XmlRootElement(name = "queue") -@XmlAccessorType(XmlAccessType.NONE) [email protected] [email protected] -public class TimelineQueue extends HierarchicalTimelineEntity { - public TimelineQueue() { - super(TimelineEntityType.YARN_QUEUE.toString()); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0bc3390/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineUser.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineUser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineUser.java deleted file mode 100644 index 45cf48f..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineUser.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.api.records.timelineservice; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlRootElement; - -@XmlRootElement(name = "user") -@XmlAccessorType(XmlAccessType.NONE) [email protected] [email protected] -public class TimelineUser extends TimelineEntity { - public TimelineUser() { - super(TimelineEntityType.YARN_USER.toString()); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0bc3390/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/UserEntity.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/UserEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/UserEntity.java new file mode 100644 index 0000000..a229fd8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/UserEntity.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.records.timelineservice; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + [email protected] [email protected] +public class UserEntity extends TimelineEntity { + public UserEntity() { + super(TimelineEntityType.YARN_USER.toString()); + } + + public UserEntity(TimelineEntity entity) { + super(entity); + if (!entity.getType().equals(TimelineEntityType.YARN_USER.toString())) { + throw new IllegalArgumentException("Incompatible entity type: " + getId()); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0bc3390/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java index 4f8ab94..caa3f3f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java @@ -23,8 +23,12 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import org.junit.Test; +import org.junit.Assert; + +import java.util.Collections; public class TestTimelineServiceRecords { @@ -87,10 +91,10 @@ public class TestTimelineServiceRecords { @Test public void testFirstClassCitizenEntities() throws Exception { - TimelineUser user = new TimelineUser(); + UserEntity user = new UserEntity(); user.setId("test user id"); - TimelineQueue queue = new TimelineQueue(); + QueueEntity queue = new QueueEntity(); queue.setId("test queue id"); @@ -98,20 +102,26 @@ public class TestTimelineServiceRecords { cluster.setId("test cluster id"); FlowEntity flow1 = new FlowEntity(); - flow1.setId("test flow id"); + //flow1.setId("test flow id 1"); flow1.setUser(user.getId()); - flow1.setVersion("test flow version"); - flow1.setRun("test run 1"); + flow1.setName("test flow name 1"); + flow1.setVersion("test flow version 1"); + flow1.setRunId(1L); FlowEntity flow2 = new FlowEntity(); - flow2.setId("test flow run id2"); + //flow2.setId("test flow run id 2"); flow2.setUser(user.getId()); - flow1.setVersion("test flow version2"); - flow2.setRun("test run 2"); + flow2.setName("test flow name 2"); + flow2.setVersion("test flow version 2"); + flow2.setRunId(2L); + + ApplicationEntity app1 = new ApplicationEntity(); + app1.setId(ApplicationId.newInstance(0, 1).toString()); + app1.setQueue(queue.getId()); - ApplicationEntity app = new ApplicationEntity(); - app.setId(ApplicationId.newInstance(0, 1).toString()); - app.setQueue(queue.getId()); + ApplicationEntity app2 = new ApplicationEntity(); + app2.setId(ApplicationId.newInstance(0, 2).toString()); + app2.setQueue(queue.getId()); ApplicationAttemptEntity appAttempt = new ApplicationAttemptEntity(); appAttempt.setId(ApplicationAttemptId.newInstance( @@ -127,12 +137,14 @@ public class TestTimelineServiceRecords { .setParent(TimelineEntityType.YARN_CLUSTER.toString(), cluster.getId()); flow1.addChild(TimelineEntityType.YARN_FLOW.toString(), flow2.getId()); flow2.setParent(TimelineEntityType.YARN_FLOW.toString(), flow1.getId()); - flow2.addChild(TimelineEntityType.YARN_APPLICATION.toString(), app.getId()); - app.setParent(TimelineEntityType.YARN_FLOW.toString(), flow2.getId()); - app.addChild(TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString(), + flow2.addChild(TimelineEntityType.YARN_APPLICATION.toString(), app1.getId()); + flow2.addChild(TimelineEntityType.YARN_APPLICATION.toString(), app2.getId()); + app1.setParent(TimelineEntityType.YARN_FLOW.toString(), flow2.getId()); + app1.addChild(TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString(), appAttempt.getId()); appAttempt - .setParent(TimelineEntityType.YARN_APPLICATION.toString(), app.getId()); + .setParent(TimelineEntityType.YARN_APPLICATION.toString(), app1.getId()); + app2.setParent(TimelineEntityType.YARN_FLOW.toString(), flow2.getId()); appAttempt.addChild(TimelineEntityType.YARN_CONTAINER.toString(), container.getId()); container.setParent(TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString(), @@ -141,14 +153,57 @@ public class TestTimelineServiceRecords { LOG.info(TimelineUtils.dumpTimelineRecordtoJSON(cluster, true)); LOG.info(TimelineUtils.dumpTimelineRecordtoJSON(flow1, true)); LOG.info(TimelineUtils.dumpTimelineRecordtoJSON(flow2, true)); - LOG.info(TimelineUtils.dumpTimelineRecordtoJSON(app, true)); + LOG.info(TimelineUtils.dumpTimelineRecordtoJSON(app1, true)); + LOG.info(TimelineUtils.dumpTimelineRecordtoJSON(app2, true)); LOG.info(TimelineUtils.dumpTimelineRecordtoJSON(appAttempt, true)); LOG.info(TimelineUtils.dumpTimelineRecordtoJSON(container, true)); + + + // Check parent/children APIs + Assert.assertNotNull(app1.getParent()); + Assert.assertEquals(flow2.getType(), app1.getParent().getType()); + Assert.assertEquals(flow2.getId(), app1.getParent().getId()); + app1.addInfo(ApplicationEntity.PARENT_INFO_KEY, "invalid parent object"); + try { + app1.getParent(); + Assert.fail(); + } catch (Exception e) { + Assert.assertTrue(e instanceof YarnRuntimeException); + Assert.assertTrue(e.getMessage().contains( + "Parent info is invalid identifier object")); + } + + Assert.assertNotNull(app1.getChildren()); + Assert.assertEquals(1, app1.getChildren().size()); + Assert.assertEquals( + appAttempt.getType(), app1.getChildren().iterator().next().getType()); + Assert.assertEquals( + appAttempt.getId(), app1.getChildren().iterator().next().getId()); + app1.addInfo(ApplicationEntity.CHILDREN_INFO_KEY, + Collections.singletonList("invalid children set")); + try { + app1.getChildren(); + Assert.fail(); + } catch (Exception e) { + Assert.assertTrue(e instanceof YarnRuntimeException); + Assert.assertTrue(e.getMessage().contains( + "Children info is invalid identifier set")); + } + app1.addInfo(ApplicationEntity.CHILDREN_INFO_KEY, + Collections.singleton("invalid child object")); + try { + app1.getChildren(); + Assert.fail(); + } catch (Exception e) { + Assert.assertTrue(e instanceof YarnRuntimeException); + Assert.assertTrue(e.getMessage().contains( + "Children info contains invalid identifier object")); + } } @Test public void testUser() throws Exception { - TimelineUser user = new TimelineUser(); + UserEntity user = new UserEntity(); user.setId("test user id"); user.addInfo("test info key 1", "test info value 1"); user.addInfo("test info key 2", "test info value 2"); @@ -157,7 +212,7 @@ public class TestTimelineServiceRecords { @Test public void testQueue() throws Exception { - TimelineQueue queue = new TimelineQueue(); + QueueEntity queue = new QueueEntity(); queue.setId("test queue id"); queue.addInfo("test info key 1", "test info value 1"); queue.addInfo("test info key 2", "test info value 2"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0bc3390/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java index 0bdb68a..044148a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java @@ -24,9 +24,12 @@ import static org.mockito.Mockito.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ExitUtil; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.timelineservice.*; import org.apache.hadoop.yarn.client.api.TimelineClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -85,6 +88,45 @@ public class TestTimelineServiceClientIntegration { } } + @Test + public void testPutExtendedEntities() throws Exception { + ApplicationId appId = ApplicationId.newInstance(0, 1); + TimelineClient client = + TimelineClient.createTimelineClient(appId); + try { + // set the timeline service address manually + client.setTimelineServiceAddress( + collectorManager.getRestServerBindAddress()); + client.init(new YarnConfiguration()); + client.start(); + ClusterEntity cluster = new ClusterEntity(); + cluster.setId(YarnConfiguration.DEFAULT_RM_CLUSTER_ID); + FlowEntity flow = new FlowEntity(); + flow.setUser(UserGroupInformation.getCurrentUser().getShortUserName()); + flow.setName("test_flow_name"); + flow.setVersion("test_flow_version"); + flow.setRunId(1L); + flow.setParent(cluster.getType(), cluster.getId()); + ApplicationEntity app = new ApplicationEntity(); + app.setId(appId.toString()); + flow.addChild(app.getType(), app.getId()); + ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1); + ApplicationAttemptEntity appAttempt = new ApplicationAttemptEntity(); + appAttempt.setId(attemptId.toString()); + ContainerId containerId = ContainerId.newContainerId(attemptId, 1); + ContainerEntity container = new ContainerEntity(); + container.setId(containerId.toString()); + UserEntity user = new UserEntity(); + user.setId(UserGroupInformation.getCurrentUser().getShortUserName()); + QueueEntity queue = new QueueEntity(); + queue.setId("default_queue"); + client.putEntities(cluster, flow, app, appAttempt, container, user, queue); + client.putEntitiesAsync(cluster, flow, app, appAttempt, container, user, queue); + } finally { + client.stop(); + } + } + private static class MockNodeTimelineCollectorManager extends NodeTimelineCollectorManager { public MockNodeTimelineCollectorManager() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0bc3390/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java index 2165c66..42fa365 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java @@ -43,7 +43,16 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationAttemptEntity; +import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity; +import org.apache.hadoop.yarn.api.records.timelineservice.ClusterEntity; +import org.apache.hadoop.yarn.api.records.timelineservice.ContainerEntity; +import org.apache.hadoop.yarn.api.records.timelineservice.FlowEntity; +import org.apache.hadoop.yarn.api.records.timelineservice.QueueEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities; +import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; +import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType; +import org.apache.hadoop.yarn.api.records.timelineservice.UserEntity; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.ForbiddenException; import org.apache.hadoop.yarn.webapp.NotFoundException; @@ -142,7 +151,8 @@ public class TimelineCollectorWebService { LOG.error("Application: "+ appId + " is not found"); throw new NotFoundException(); // different exception? } - collector.putEntities(entities, callerUgi); + + collector.putEntities(processTimelineEntities(entities), callerUgi); return Response.ok().build(); } catch (Exception e) { LOG.error("Error putting entities", e); @@ -151,7 +161,7 @@ public class TimelineCollectorWebService { } } - private ApplicationId parseApplicationId(String appId) { + private static ApplicationId parseApplicationId(String appId) { try { if (appId != null) { return ConverterUtils.toApplicationId(appId.trim()); @@ -159,15 +169,16 @@ public class TimelineCollectorWebService { return null; } } catch (Exception e) { + LOG.error("Invalid application ID: " + appId); return null; } } - private void init(HttpServletResponse response) { + private static void init(HttpServletResponse response) { response.setContentType(null); } - private UserGroupInformation getUser(HttpServletRequest req) { + private static UserGroupInformation getUser(HttpServletRequest req) { String remoteUser = req.getRemoteUser(); UserGroupInformation callerUgi = null; if (remoteUser != null) { @@ -175,4 +186,50 @@ public class TimelineCollectorWebService { } return callerUgi; } + + // The process may not be necessary according to the way we write the backend, + // but let's keep it for now in case we need to use sub-classes APIs in the + // future (e.g., aggregation). + private static TimelineEntities processTimelineEntities( + TimelineEntities entities) { + TimelineEntities entitiesToReturn = new TimelineEntities(); + for (TimelineEntity entity : entities.getEntities()) { + TimelineEntityType type = null; + try { + type = TimelineEntityType.valueOf(entity.getType()); + } catch (IllegalArgumentException e) { + type = null; + } + if (type != null) { + switch (type) { + case YARN_CLUSTER: + entitiesToReturn.addEntity(new ClusterEntity(entity)); + break; + case YARN_FLOW: + entitiesToReturn.addEntity(new FlowEntity(entity)); + break; + case YARN_APPLICATION: + entitiesToReturn.addEntity(new ApplicationEntity(entity)); + break; + case YARN_APPLICATION_ATTEMPT: + entitiesToReturn.addEntity(new ApplicationAttemptEntity(entity)); + break; + case YARN_CONTAINER: + entitiesToReturn.addEntity(new ContainerEntity(entity)); + break; + case YARN_QUEUE: + entitiesToReturn.addEntity(new QueueEntity(entity)); + break; + case YARN_USER: + entitiesToReturn.addEntity(new UserEntity(entity)); + break; + default: + break; + } + } else { + entitiesToReturn.addEntity(entity); + } + } + return entitiesToReturn; + } }
