Repository: tez Updated Branches: refs/heads/branch-0.7 4b8518aad -> f167c8575
TEZ-3149. Tez-tools: Add username in DagInfo (rbalamohan) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/f167c857 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/f167c857 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/f167c857 Branch: refs/heads/branch-0.7 Commit: f167c8575eb78acecce5e42fcd96fbf7394f1ce7 Parents: 4b8518a Author: Rajesh Balamohan <[email protected]> Authored: Tue Mar 1 03:29:14 2016 +0530 Committer: Rajesh Balamohan <[email protected]> Committed: Tue Mar 1 03:29:14 2016 +0530 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../tez/history/parser/SimpleHistoryParser.java | 16 ++++++++++++++++ .../tez/history/parser/datamodel/BaseParser.java | 5 +++++ .../tez/history/parser/datamodel/DagInfo.java | 10 ++++++++++ .../org/apache/tez/history/TestHistoryParser.java | 2 ++ 5 files changed, 34 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/f167c857/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 96ab0ac..c9b13b0 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -9,6 +9,7 @@ INCOMPATIBLE CHANGES TEZ-2972. Avoid task rescheduling when a node turns unhealthy ALL CHANGES: + TEZ-3149. Tez-tools: Add username in DagInfo TEZ-2988. DAGAppMaster::shutdownTezAM should return with a no-op if it has been invoked earlier. TEZ-3147. Intermediate mem-to-mem: Fix early exit when only one segment can fit into memory TEZ-1911. MergeManager's unconditionalReserve() should check for memory limits before allocating. http://git-wip-us.apache.org/repos/asf/tez/blob/f167c857/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/SimpleHistoryParser.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/SimpleHistoryParser.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/SimpleHistoryParser.java index 4d3e96f..bd23717 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/SimpleHistoryParser.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/SimpleHistoryParser.java @@ -114,6 +114,7 @@ public class SimpleHistoryParser extends BaseParser { Map<String, JSONObject> taskJsonMap = Maps.newHashMap(); Map<String, JSONObject> attemptJsonMap = Maps.newHashMap(); TezDAGID tezDAGID = TezDAGID.fromString(dagId); + String userName = null; while (scanner.hasNext()) { String line = scanner.next(); JSONObject jsonObject = new JSONObject(line); @@ -132,6 +133,20 @@ public class SimpleHistoryParser extends BaseParser { } JSONObject otherInfo = jsonObject.optJSONObject(Constants.OTHER_INFO); JSONObject dagOtherInfo = dagJson.getJSONObject(Constants.OTHER_INFO); + JSONArray relatedEntities = dagJson.optJSONArray(Constants + .RELATED_ENTITIES); + //UserName is present in related entities + // {"entity":"userXYZ","entitytype":"user"} + if (relatedEntities != null) { + for (int i = 0; i < relatedEntities.length(); i++) { + JSONObject subEntity = relatedEntities.getJSONObject(i); + String subEntityType = subEntity.optString(Constants.ENTITY_TYPE); + if (subEntityType != null && subEntityType.equals(Constants.USER)) { + userName = subEntity.getString(Constants.ENTITY); + break; + } + } + } populateOtherInfo(otherInfo, dagOtherInfo); } else if (entityType.equals(Constants.TEZ_VERTEX_ID)) { @@ -177,6 +192,7 @@ public class SimpleHistoryParser extends BaseParser { scanner.close(); if (dagJson != null) { this.dagInfo = DagInfo.create(dagJson); + setUserName(userName); } else { LOG.error("Dag is not yet parsed. Looks like partial file."); throw new TezException( http://git-wip-us.apache.org/repos/asf/tez/blob/f167c857/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/BaseParser.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/BaseParser.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/BaseParser.java index 62ba474..84fd976 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/BaseParser.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/BaseParser.java @@ -111,4 +111,9 @@ public abstract class BaseParser { } }); } + + protected void setUserName(String userName) { + Preconditions.checkArgument(dagInfo != null, "DagInfo can not be null"); + dagInfo.setUserName(userName); + } } http://git-wip-us.apache.org/repos/asf/tez/blob/f167c857/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java index 8057be7..228a159 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java @@ -66,6 +66,7 @@ public class DagInfo extends BaseInfo { private final int numVertices; private final String status; private final String diagnostics; + private String userName; private VersionInfo versionInfo; private CallerContext callerContext; @@ -102,6 +103,7 @@ public class DagInfo extends BaseInfo { long sTime = otherInfoNode.optLong(Constants.START_TIME); long eTime= otherInfoNode.optLong(Constants.FINISH_TIME); + userName = otherInfoNode.optString(Constants.USER); if (eTime < sTime) { LOG.warn("DAG has got wrong start/end values. " + "startTime=" + sTime + ", endTime=" + eTime + ". Will check " @@ -612,4 +614,12 @@ public class DagInfo extends BaseInfo { return failedTasks; } + public final String getUserName() { + return userName; + } + + final void setUserName(String userName) { + this.userName = userName; + } + } http://git-wip-us.apache.org/repos/asf/tez/blob/f167c857/tez-plugins/tez-history-parser/src/test/java/org/apache/tez/history/TestHistoryParser.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-history-parser/src/test/java/org/apache/tez/history/TestHistoryParser.java b/tez-plugins/tez-history-parser/src/test/java/org/apache/tez/history/TestHistoryParser.java index 372585b..a814425 100644 --- a/tez-plugins/tez-history-parser/src/test/java/org/apache/tez/history/TestHistoryParser.java +++ b/tez-plugins/tez-history-parser/src/test/java/org/apache/tez/history/TestHistoryParser.java @@ -697,6 +697,8 @@ public class TestHistoryParser { assertTrue(versionInfo.getBuildTime() != null); } + assertTrue(dagInfo.getUserName() != null); + assertTrue(!dagInfo.getUserName().isEmpty()); assertTrue(dagInfo.getStartTime() > 0); assertTrue(dagInfo.getFinishTimeInterval() > 0); assertTrue(dagInfo.getStartTimeInterval() == 0);
