Repository: tez Updated Branches: refs/heads/master ccd0e04fe -> bcdef6605
TEZ-3327. ATS Parser: Populate config details available in dag (rbalamohan) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/bcdef660 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/bcdef660 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/bcdef660 Branch: refs/heads/master Commit: bcdef660558d41b8f466417b52a08b037ba1e4a5 Parents: ccd0e04 Author: Rajesh Balamohan <[email protected]> Authored: Fri Jul 8 03:54:31 2016 +0530 Committer: Rajesh Balamohan <[email protected]> Committed: Fri Jul 8 03:54:31 2016 +0530 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../org/apache/tez/history/parser/ATSFileParser.java | 12 +++++++++++- .../tez/history/parser/datamodel/BaseParser.java | 3 +++ .../apache/tez/history/parser/datamodel/DagInfo.java | 9 +++++++++ .../java/org/apache/tez/history/TestHistoryParser.java | 13 +++++++++++++ 5 files changed, 37 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/bcdef660/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index c973ad8..02334b4 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3327. ATS Parser: Populate config details available in dag. TEZ-3325. Flaky test in TestDAGImpl.testCounterLimits. TEZ-3323. Update licese and notice for xml-apis jar. Also update year in notice to 2016. TEZ-3223. Support a NullHistoryLogger to disable history logging if needed. http://git-wip-us.apache.org/repos/asf/tez/blob/bcdef660/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/ATSFileParser.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/ATSFileParser.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/ATSFileParser.java index a789536..89db976 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/ATSFileParser.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/ATSFileParser.java @@ -19,6 +19,7 @@ package org.apache.tez.history.parser; import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; import org.apache.commons.io.IOUtils; import org.apache.tez.dag.api.TezException; import org.apache.tez.history.parser.datamodel.BaseParser; @@ -39,6 +40,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.util.Enumeration; +import java.util.Iterator; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; @@ -158,7 +160,15 @@ public class ATSFileParser extends BaseParser implements ATSData { String revision = tezVersion.optString(Constants.REVISION); this.versionInfo = new VersionInfo(version, revision, buildTime); } - //TODO: might need to parse config info? (e.g, hive settings etc. could consume memory) + //Parse Config info + this.config = Maps.newHashMap(); + JSONObject configNode = otherInfoNode.getJSONObject(Constants.CONFIG); + Iterator it = configNode.keys(); + while(it.hasNext()) { + String key = (String) it.next(); + String value = configNode.getString(key); + config.put(key, value); + } } LOG.debug("Finished parsing tez application"); } http://git-wip-us.apache.org/repos/asf/tez/blob/bcdef660/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/BaseParser.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/BaseParser.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/BaseParser.java index 84fd976..362dbd9 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/BaseParser.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/BaseParser.java @@ -26,11 +26,13 @@ import org.apache.tez.dag.records.TezTaskAttemptID; import org.apache.tez.dag.records.TezTaskID; import java.util.List; +import java.util.Map; public abstract class BaseParser { protected DagInfo dagInfo; protected VersionInfo versionInfo; + protected Map<String, String> config; protected final List<VertexInfo> vertexList; protected final List<TaskInfo> taskList; protected final List<TaskAttemptInfo> attemptList; @@ -95,6 +97,7 @@ public abstract class BaseParser { } dagInfo.setVersionInfo(versionInfo); + dagInfo.setAppConfig(config); } /** http://git-wip-us.apache.org/repos/asf/tez/blob/bcdef660/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java index 228a159..6bd691c 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java @@ -83,6 +83,7 @@ public class DagInfo extends BaseInfo { private Map<String, VertexInfo> vertexNameMap; private Multimap<Container, TaskAttemptInfo> containerMapping; + private Map<String, String> config; DagInfo(JSONObject jsonObject) throws JSONException { super(jsonObject); @@ -350,6 +351,14 @@ public class DagInfo extends BaseInfo { vertexNameMap.put(vertexInfo.getVertexName(), vertexInfo); } + void setAppConfig(Map<String, String> config) { + this.config = config; + } + + public Map<String, String> getAppConfig() { + return (config != null) ? Collections.unmodifiableMap(config) : null; + } + void setVersionInfo(VersionInfo versionInfo) { this.versionInfo = versionInfo; } http://git-wip-us.apache.org/repos/asf/tez/blob/bcdef660/tez-plugins/tez-history-parser/src/test/java/org/apache/tez/history/TestHistoryParser.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-history-parser/src/test/java/org/apache/tez/history/TestHistoryParser.java b/tez-plugins/tez-history-parser/src/test/java/org/apache/tez/history/TestHistoryParser.java index 401a606..92c4ad8 100644 --- a/tez-plugins/tez-history-parser/src/test/java/org/apache/tez/history/TestHistoryParser.java +++ b/tez-plugins/tez-history-parser/src/test/java/org/apache/tez/history/TestHistoryParser.java @@ -210,6 +210,7 @@ public class TestHistoryParser { DagInfo dagInfoFromATS = getDagInfo(dagId); verifyDagInfo(dagInfoFromATS, true); verifyJobSpecificInfo(dagInfoFromATS); + checkConfig(dagInfoFromATS); //Now run with SimpleHistoryLogging dagId = runWordCount(WordCount.TokenProcessor.class.getName(), @@ -244,6 +245,16 @@ public class TestHistoryParser { return dagInfo; } + private void checkConfig(DagInfo dagInfo) { + assertTrue("DagInfo is " + dagInfo, dagInfo != null); + //Check configs + assertTrue("DagInfo config size=" + dagInfo.getAppConfig().size(), + dagInfo.getAppConfig().size() > 0); + //Sample config element + assertTrue("DagInfo config=" + dagInfo.getAppConfig(), + Integer.parseInt(dagInfo.getAppConfig().get("dfs.replication")) > 0); + } + private void verifyJobSpecificInfo(DagInfo dagInfo) { //Job specific assertTrue(dagInfo.getNumVertices() == 2); @@ -373,6 +384,8 @@ public class TestHistoryParser { //Parse ATS data DagInfo dagInfo = getDagInfo(dagId); + //Applicable for ATS dataset + checkConfig(dagInfo); //Verify DAGInfo. Verifies vertex, task, taskAttempts in recursive manner verifyDagInfo(dagInfo, true);
