HIVE-13407 : Add more subtlety to TezCompiler Perf Logging (Hari Sankar Subramaniyan via Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a2b7b4d9 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a2b7b4d9 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a2b7b4d9 Branch: refs/heads/llap Commit: a2b7b4d9dca82aa599b4b15fa7927f1d6403da79 Parents: 0ef92b2 Author: Hari Subramaniyan <[email protected]> Authored: Thu Apr 21 13:57:00 2016 -0800 Committer: Ashutosh Chauhan <[email protected]> Committed: Fri Apr 22 17:57:20 2016 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hive/ql/log/PerfLogger.java | 1 + .../apache/hadoop/hive/ql/parse/TezCompiler.java | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/a2b7b4d9/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java index 6f9347c..87b82f8 100644 --- a/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java +++ b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java @@ -63,6 +63,7 @@ public class PerfLogger { public static final String FAILURE_HOOK = "FailureHook."; public static final String DRIVER_RUN = "Driver.run"; public static final String TIME_TO_SUBMIT = "TimeToSubmit"; + public static final String TEZ_COMPILER = "TezCompiler"; public static final String TEZ_SUBMIT_TO_RUNNING = "TezSubmitToRunningDag"; public static final String TEZ_BUILD_DAG = "TezBuildDag"; public static final String TEZ_SUBMIT_DAG = "TezSubmitDag"; http://git-wip-us.apache.org/repos/asf/hive/blob/a2b7b4d9/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java index ff971ac..8e67ce8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java @@ -115,23 +115,28 @@ public class TezCompiler extends TaskCompiler { protected void optimizeOperatorPlan(ParseContext pCtx, Set<ReadEntity> inputs, Set<WriteEntity> outputs) throws SemanticException { PerfLogger perfLogger = SessionState.getPerfLogger(); - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); // Create the context for the walker OptimizeTezProcContext procCtx = new OptimizeTezProcContext(conf, pCtx, inputs, outputs); - + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER); // setup dynamic partition pruning where possible runDynamicPartitionPruning(procCtx, inputs, outputs); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Setup dynamic partition pruning"); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER); // setup stats in the operator plan runStatsAnnotation(procCtx); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Setup stats in the operator plan"); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER); // run the optimizations that use stats for optimization runStatsDependentOptimizations(procCtx, inputs, outputs); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Run the optimizations that use stats for optimization"); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER); // after the stats phase we might have some cyclic dependencies that we need // to take care of. runCycleAnalysisForPartitionPruning(procCtx, inputs, outputs); - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Tez compiler"); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Run cycle analysis for partition pruning"); } @@ -323,7 +328,8 @@ public class TezCompiler extends TaskCompiler { List<Task<MoveWork>> mvTask, Set<ReadEntity> inputs, Set<WriteEntity> outputs) throws SemanticException { - + PerfLogger perfLogger = SessionState.getPerfLogger(); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER); ParseContext tempParseContext = getParseContext(pCtx, rootTasks); GenTezUtils utils = new GenTezUtils(); GenTezWork genTezWork = new GenTezWork(utils); @@ -389,6 +395,7 @@ public class TezCompiler extends TaskCompiler { LOG.debug("Handling AppMasterEventOperator: " + event); GenTezUtils.processAppMasterEvent(procCtx, event); } + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "generateTaskTree"); } @Override @@ -449,6 +456,8 @@ public class TezCompiler extends TaskCompiler { @Override protected void optimizeTaskPlan(List<Task<? extends Serializable>> rootTasks, ParseContext pCtx, Context ctx) throws SemanticException { + PerfLogger perfLogger = SessionState.getPerfLogger(); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER); PhysicalContext physicalCtx = new PhysicalContext(conf, pCtx, pCtx.getContext(), rootTasks, pCtx.getFetchTask()); @@ -498,7 +507,7 @@ public class TezCompiler extends TaskCompiler { // the backend. If you have a physical optimization that changes // table scans or filters, you have to invoke it before this one. physicalCtx = new SerializeFilter().resolve(physicalCtx); - + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "optimizeTaskPlan"); return; } }
