Repository: tez Updated Branches: refs/heads/master dd7919061 -> e9e9db1b6
TEZ-3149. Tez-tools: Add username in DagInfo (rbalamohan) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/e9e9db1b Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/e9e9db1b Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/e9e9db1b Branch: refs/heads/master Commit: e9e9db1b65912258d672cbd3e9fab933b0dda7fd Parents: dd79190 Author: Rajesh Balamohan <[email protected]> Authored: Tue Mar 1 03:40:28 2016 +0530 Committer: Rajesh Balamohan <[email protected]> Committed: Tue Mar 1 03:40:28 2016 +0530 ---------------------------------------------------------------------- CHANGES.txt | 2 ++ .../tez/history/parser/SimpleHistoryParser.java | 16 ++++++++++++++++ .../tez/history/parser/datamodel/BaseParser.java | 5 +++++ .../tez/history/parser/datamodel/DagInfo.java | 10 ++++++++++ .../org/apache/tez/history/TestHistoryParser.java | 2 ++ 5 files changed, 35 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/e9e9db1b/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 6b73fc6..8084a59 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES TEZ-3029. Add an onError method to service plugin contexts. ALL CHANGES: + TEZ-3149. Tez-tools: Add username in DagInfo. TEZ-2988. DAGAppMaster::shutdownTezAM should return with a no-op if it has been invoked earlier. TEZ-3147. Intermediate mem-to-mem: Fix early exit when only one segment can fit into memory TEZ-3141. mapreduce.task.timeout is not translated to container heartbeat timeout @@ -393,6 +394,7 @@ INCOMPATIBLE CHANGES TEZ-2949. Allow duplicate dag names within session for Tez. ALL CHANGES: + TEZ-3149. Tez-tools: Add username in DagInfo. TEZ-2988. DAGAppMaster::shutdownTezAM should return with a no-op if it has been invoked earlier. TEZ-3141. mapreduce.task.timeout is not translated to container heartbeat timeout TEZ-3129. Tez task and task attempt UI needs application fails with NotFoundException http://git-wip-us.apache.org/repos/asf/tez/blob/e9e9db1b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/SimpleHistoryParser.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/SimpleHistoryParser.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/SimpleHistoryParser.java index 863cc40..989dd51 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/SimpleHistoryParser.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/SimpleHistoryParser.java @@ -114,6 +114,7 @@ public class SimpleHistoryParser extends BaseParser { Map<String, JSONObject> taskJsonMap = Maps.newHashMap(); Map<String, JSONObject> attemptJsonMap = Maps.newHashMap(); TezDAGID tezDAGID = TezDAGID.fromString(dagId); + String userName = null; while (scanner.hasNext()) { String line = scanner.next(); JSONObject jsonObject = new JSONObject(line); @@ -133,6 +134,20 @@ public class SimpleHistoryParser extends BaseParser { } JSONObject otherInfo = jsonObject.optJSONObject(Constants.OTHER_INFO); JSONObject dagOtherInfo = dagJson.getJSONObject(Constants.OTHER_INFO); + JSONArray relatedEntities = dagJson.optJSONArray(Constants + .RELATED_ENTITIES); + //UserName is present in related entities + // {"entity":"userXYZ","entitytype":"user"} + if (relatedEntities != null) { + for (int i = 0; i < relatedEntities.length(); i++) { + JSONObject subEntity = relatedEntities.getJSONObject(i); + String subEntityType = subEntity.optString(Constants.ENTITY_TYPE); + if (subEntityType != null && subEntityType.equals(Constants.USER)) { + userName = subEntity.getString(Constants.ENTITY); + break; + } + } + } populateOtherInfo(otherInfo, dagOtherInfo); break; case Constants.TEZ_VERTEX_ID: @@ -181,6 +196,7 @@ public class SimpleHistoryParser extends BaseParser { scanner.close(); if (dagJson != null) { this.dagInfo = DagInfo.create(dagJson); + setUserName(userName); } else { LOG.error("Dag is not yet parsed. Looks like partial file."); throw new TezException( http://git-wip-us.apache.org/repos/asf/tez/blob/e9e9db1b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/BaseParser.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/BaseParser.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/BaseParser.java index 62ba474..84fd976 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/BaseParser.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/BaseParser.java @@ -111,4 +111,9 @@ public abstract class BaseParser { } }); } + + protected void setUserName(String userName) { + Preconditions.checkArgument(dagInfo != null, "DagInfo can not be null"); + dagInfo.setUserName(userName); + } } http://git-wip-us.apache.org/repos/asf/tez/blob/e9e9db1b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java index 8057be7..228a159 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java @@ -66,6 +66,7 @@ public class DagInfo extends BaseInfo { private final int numVertices; private final String status; private final String diagnostics; + private String userName; private VersionInfo versionInfo; private CallerContext callerContext; @@ -102,6 +103,7 @@ public class DagInfo extends BaseInfo { long sTime = otherInfoNode.optLong(Constants.START_TIME); long eTime= otherInfoNode.optLong(Constants.FINISH_TIME); + userName = otherInfoNode.optString(Constants.USER); if (eTime < sTime) { LOG.warn("DAG has got wrong start/end values. " + "startTime=" + sTime + ", endTime=" + eTime + ". Will check " @@ -612,4 +614,12 @@ public class DagInfo extends BaseInfo { return failedTasks; } + public final String getUserName() { + return userName; + } + + final void setUserName(String userName) { + this.userName = userName; + } + } http://git-wip-us.apache.org/repos/asf/tez/blob/e9e9db1b/tez-plugins/tez-history-parser/src/test/java/org/apache/tez/history/TestHistoryParser.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-history-parser/src/test/java/org/apache/tez/history/TestHistoryParser.java b/tez-plugins/tez-history-parser/src/test/java/org/apache/tez/history/TestHistoryParser.java index 372585b..a814425 100644 --- a/tez-plugins/tez-history-parser/src/test/java/org/apache/tez/history/TestHistoryParser.java +++ b/tez-plugins/tez-history-parser/src/test/java/org/apache/tez/history/TestHistoryParser.java @@ -697,6 +697,8 @@ public class TestHistoryParser { assertTrue(versionInfo.getBuildTime() != null); } + assertTrue(dagInfo.getUserName() != null); + assertTrue(!dagInfo.getUserName().isEmpty()); assertTrue(dagInfo.getStartTime() > 0); assertTrue(dagInfo.getFinishTimeInterval() > 0); assertTrue(dagInfo.getStartTimeInterval() == 0);
