Repository: hive Updated Branches: refs/heads/master b0e653aec -> a6435cca3
HIVE-18275 : add HS2-level WM metrics (Sergey Shelukhin, reviewed by Thejas M Nair) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a6435cca Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a6435cca Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a6435cca Branch: refs/heads/master Commit: a6435cca3ab11d39d7887eae8e077c386ea9133c Parents: b0e653a Author: sergey <[email protected]> Authored: Fri Jan 5 14:02:41 2018 -0800 Committer: sergey <[email protected]> Committed: Fri Jan 5 14:02:41 2018 -0800 ---------------------------------------------------------------------- common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java | 1 + ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java | 2 ++ .../hadoop/hive/ql/exec/tez/WorkloadManagerFederation.java | 4 ++-- .../ql/exec/tez/monitoring/QueryExecutionBreakdownSummary.java | 5 ++++- 4 files changed, 9 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/a6435cca/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java index 008b3b0..2f7ed07 100644 --- a/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java +++ b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java @@ -70,6 +70,7 @@ public class PerfLogger { public static final String TEZ_RUN_PROCESSOR = "TezRunProcessor"; public static final String TEZ_INIT_OPERATORS = "TezInitializeOperators"; public static final String LOAD_HASHTABLE = "LoadHashtable"; + public static final String TEZ_GET_SESSION = "TezGetSession"; public static final String SPARK_SUBMIT_TO_RUNNING = "SparkSubmitToRunning"; public static final String SPARK_BUILD_PLAN = "SparkBuildPlan"; http://git-wip-us.apache.org/repos/asf/hive/blob/a6435cca/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java index 1a24b44..38f80fd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java @@ -175,8 +175,10 @@ public class TezTask extends Task<TezWork> { CallerContext callerContext = CallerContext.create( "HIVE", queryPlan.getQueryId(), "HIVE_QUERY_ID", queryPlan.getQueryStr()); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TEZ_GET_SESSION); session = sessionRef.value = WorkloadManagerFederation.getSession( sessionRef.value, conf, mi, getWork().getLlapMode(), wmContext); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.TEZ_GET_SESSION); try { ss.setTezSession(session); http://git-wip-us.apache.org/repos/asf/hive/blob/a6435cca/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/WorkloadManagerFederation.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/WorkloadManagerFederation.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/WorkloadManagerFederation.java index 9d56204..4b5022a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/WorkloadManagerFederation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/WorkloadManagerFederation.java @@ -29,7 +29,7 @@ public class WorkloadManagerFederation { private static final Logger LOG = LoggerFactory.getLogger(WorkloadManagerFederation.class); public static TezSessionState getSession(TezSessionState session, HiveConf conf, - MappingInput input, boolean isUnmanagedLlapMode, final WmContext wmContext) throws Exception { + MappingInput input, boolean isUnmanagedLlapMode, WmContext wmContext) throws Exception { Set<String> desiredCounters = new HashSet<>(); // 1. If WM is not present just go to unmanaged. WorkloadManager wm = WorkloadManager.getInstance(); @@ -54,7 +54,7 @@ public class WorkloadManagerFederation { wm.updateTriggers(result); return result; } catch (WorkloadManager.NoPoolMappingException ex) { - LOG.info("NoPoolMappingException thrown. Getting an un-managed session.."); + LOG.info("NoPoolMappingException thrown. Getting an un-managed session"); return getUnmanagedSession(session, conf, desiredCounters, isUnmanagedLlapMode, wmContext); } } http://git-wip-us.apache.org/repos/asf/hive/blob/a6435cca/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/QueryExecutionBreakdownSummary.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/QueryExecutionBreakdownSummary.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/QueryExecutionBreakdownSummary.java index 271e3c6..178488f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/QueryExecutionBreakdownSummary.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/QueryExecutionBreakdownSummary.java @@ -37,12 +37,14 @@ class QueryExecutionBreakdownSummary implements PrintSummary { private final Long compileEndTime; private final Long dagSubmitStartTime; private final Long submitToRunningDuration; + private final Long getSessionDuration; QueryExecutionBreakdownSummary(PerfLogger perfLogger) { this.perfLogger = perfLogger; this.compileEndTime = perfLogger.getEndTime(PerfLogger.COMPILE); this.dagSubmitStartTime = perfLogger.getStartTime(PerfLogger.TEZ_SUBMIT_DAG); this.submitToRunningDuration = perfLogger.getDuration(PerfLogger.TEZ_SUBMIT_TO_RUNNING); + this.getSessionDuration = perfLogger.getDuration(PerfLogger.TEZ_GET_SESSION); } private String formatNumber(long number) { @@ -66,8 +68,9 @@ class QueryExecutionBreakdownSummary implements PrintSummary { console.printInfo(format("Compile Query", compile)); // prepare plan for submission (building DAG, adding resources, creating scratch dirs etc.) - long totalDAGPrep = dagSubmitStartTime - compileEndTime; + long totalDAGPrep = dagSubmitStartTime - compileEndTime - getSessionDuration; console.printInfo(format("Prepare Plan", totalDAGPrep)); + console.printInfo(format("Get Query Coordinator (AM)", getSessionDuration)); // submit to accept dag (if session is closed, this will include re-opening of session time, // localizing files for AM, submitting DAG)
