HIVE-12952 : Show query sub-pages on webui (Szehon, reviewed by Aihua Xu and Lenni Kuff)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2d94c0b0 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2d94c0b0 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2d94c0b0 Branch: refs/heads/master Commit: 2d94c0b0bd9fe88747eb749dba8e1ab51d94a4ff Parents: f2e9edb Author: Szehon Ho <sze...@cloudera.com> Authored: Wed Feb 3 22:18:49 2016 +0100 Committer: Szehon Ho <sze...@cloudera.com> Committed: Wed Feb 3 22:18:49 2016 +0100 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/conf/HiveConf.java | 15 + .../apache/hadoop/hive/ql/log/PerfLogger.java | 9 + .../java/org/apache/hive/http/HttpServer.java | 2 +- pom.xml | 12 + .../java/org/apache/hadoop/hive/ql/Driver.java | 48 +- .../org/apache/hadoop/hive/ql/QueryDisplay.java | 253 +++ .../apache/hadoop/hive/ql/metadata/Hive.java | 6 +- service/pom.xml | 51 +- .../org/apache/hive/tmpl/QueryProfileTmpl.jamon | 298 +++ .../hive/service/cli/operation/Operation.java | 4 + .../service/cli/operation/OperationManager.java | 82 +- .../service/cli/operation/SQLOperation.java | 39 +- .../cli/operation/SQLOperationDisplay.java | 99 + .../cli/operation/SQLOperationDisplayCache.java | 39 + .../service/cli/operation/SQLOperationInfo.java | 48 - .../apache/hive/service/server/HiveServer2.java | 2 + .../service/servlet/QueryProfileServlet.java | 49 + .../hive-webapps/hiveserver2/hiveserver2.jsp | 72 +- .../hive-webapps/static/js/bootstrap.js | 1999 ++++++++++++++++++ .../hive-webapps/static/js/bootstrap.min.js | 6 + .../hive-webapps/static/js/jquery.min.js | 2 + .../src/resources/hive-webapps/static/js/tab.js | 38 + 22 files changed, 3028 insertions(+), 145 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 6678de6..73e6c21 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -3741,6 +3741,21 @@ public class HiveConf extends Configuration { } } + /** + * @return true if HS2 webui is enabled + */ + public boolean isWebUiEnabled() { + return this.getIntVar(ConfVars.HIVE_SERVER2_WEBUI_PORT) != 0; + } + + /** + * @return true if HS2 webui query-info cache is enabled + */ + public boolean isWebUiQueryInfoCacheEnabled() { + return isWebUiEnabled() && this.getIntVar(ConfVars.HIVE_SERVER2_WEBUI_MAX_HISTORIC_QUERIES) > 0; + } + + public static boolean isLoadMetastoreConfig() { return loadMetastoreConfig; } http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java index d4194cf..8fa5cbf 100644 --- a/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java +++ b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.log; +import com.google.common.collect.ImmutableMap; import org.apache.hadoop.hive.common.metrics.common.Metrics; import org.apache.hadoop.hive.common.metrics.common.MetricsFactory; import org.apache.hadoop.hive.conf.HiveConf; @@ -222,4 +223,12 @@ public class PerfLogger { LOG.warn("Error recording metrics", e); } } + + public ImmutableMap<String, Long> getStartTimes() { + return ImmutableMap.copyOf(startTimes); + } + + public ImmutableMap<String, Long> getEndTimes() { + return ImmutableMap.copyOf(endTimes); + } } http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/common/src/java/org/apache/hive/http/HttpServer.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hive/http/HttpServer.java b/common/src/java/org/apache/hive/http/HttpServer.java index 9e23b11..b8836de 100644 --- a/common/src/java/org/apache/hive/http/HttpServer.java +++ b/common/src/java/org/apache/hive/http/HttpServer.java @@ -435,7 +435,7 @@ public class HttpServer { * @param pathSpec The path spec for the servlet * @param clazz The servlet class */ - void addServlet(String name, String pathSpec, + public void addServlet(String name, String pathSpec, Class<? extends HttpServlet> clazz) { ServletHolder holder = new ServletHolder(clazz); if (name != null) { http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 8c2257f..3c06188 100644 --- a/pom.xml +++ b/pom.xml @@ -140,6 +140,8 @@ <!-- jackson 1 and 2 lines can coexist without issue, as they have different artifactIds --> <jackson.new.version>2.4.2</jackson.new.version> <jasper.version>5.5.23</jasper.version> + <jamon.plugin.version>2.3.4</jamon.plugin.version> + <jamon-runtime.version>2.3.1</jamon-runtime.version> <javaewah.version>0.3.2</javaewah.version> <javax-servlet.version>3.0.0.v201112011016</javax-servlet.version> <javolution.version>5.5.1</javolution.version> @@ -725,6 +727,11 @@ </exclusion> </exclusions> </dependency> + <dependency> + <groupId>org.jamon</groupId> + <artifactId>jamon-runtime</artifactId> + <version>${jamon-runtime.version}</version> + </dependency> </dependencies> </dependencyManagement> @@ -1064,6 +1071,11 @@ </excludes> </configuration> </plugin> + <plugin> + <groupId>org.jamon</groupId> + <artifactId>jamon-maven-plugin</artifactId> + <version>${jamon.plugin.version}</version> + </plugin> </plugins> </build> http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/ql/src/java/org/apache/hadoop/hive/ql/Driver.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java index bcf62a7..7147a9a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -36,6 +36,7 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.mapreduce.MRJobConfig; @@ -53,7 +54,6 @@ import org.apache.hadoop.hive.metastore.api.Schema; import org.apache.hadoop.hive.ql.exec.ConditionalTask; import org.apache.hadoop.hive.ql.exec.ExplainTask; import org.apache.hadoop.hive.ql.exec.FetchTask; -import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; @@ -159,7 +159,7 @@ public class Driver implements CommandProcessor { private String operationId; // For WebUI. Kept alive after queryPlan is freed. - private String savedQueryString; + private final QueryDisplay queryDisplay = new QueryDisplay(); private boolean checkConcurrency() { boolean supportConcurrency = conf.getBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY); @@ -387,7 +387,6 @@ public class Driver implements CommandProcessor { } catch (Exception e) { LOG.warn("WARNING! Query command could not be redacted." + e); } - this.savedQueryString = queryStr; //holder for parent command type/string when executing reentrant queries QueryState queryState = new QueryState(); @@ -409,6 +408,10 @@ public class Driver implements CommandProcessor { conf.setVar(HiveConf.ConfVars.HIVEQUERYID, queryId); } + //save some info for webUI for use after plan is freed + this.queryDisplay.setQueryStr(queryStr); + this.queryDisplay.setQueryId(queryId); + LOG.info("Compiling command(queryId=" + queryId + "): " + queryStr); SessionState.get().setupQueryCurrentTimestamp(); @@ -519,11 +522,17 @@ public class Driver implements CommandProcessor { } } - if (conf.getBoolVar(ConfVars.HIVE_LOG_EXPLAIN_OUTPUT)) { + if (conf.getBoolVar(ConfVars.HIVE_LOG_EXPLAIN_OUTPUT) || + conf.isWebUiQueryInfoCacheEnabled()) { String explainOutput = getExplainOutput(sem, plan, tree); if (explainOutput != null) { - LOG.info("EXPLAIN output for queryid " + queryId + " : " + if (conf.getBoolVar(ConfVars.HIVE_LOG_EXPLAIN_OUTPUT)) { + LOG.info("EXPLAIN output for queryid " + queryId + " : " + explainOutput); + } + if (conf.isWebUiQueryInfoCacheEnabled()) { + queryDisplay.setExplainPlan(explainOutput); + } } } return 0; @@ -553,18 +562,20 @@ public class Driver implements CommandProcessor { // since it exceeds valid range of shell return values } finally { double duration = perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.COMPILE)/1000.00; - dumpMetaCallTimingWithoutEx("compilation"); + ImmutableMap<String, Long> compileHMSTimings = dumpMetaCallTimingWithoutEx("compilation"); + queryDisplay.setHmsTimings(QueryDisplay.Phase.COMPILATION, compileHMSTimings); restoreSession(queryState); LOG.info("Completed compiling command(queryId=" + queryId + "); Time taken: " + duration + " seconds"); } } - private void dumpMetaCallTimingWithoutEx(String phase) { + private ImmutableMap<String, Long> dumpMetaCallTimingWithoutEx(String phase) { try { - Hive.get().dumpAndClearMetaCallTiming(phase); + return Hive.get().dumpAndClearMetaCallTiming(phase); } catch (HiveException he) { LOG.warn("Caught exception attempting to write metadata call information " + he, he); } + return null; } /** @@ -1186,6 +1197,13 @@ public class Driver implements CommandProcessor { + org.apache.hadoop.util.StringUtils.stringifyException(e)); } } + + //Save compile-time PerfLogging for WebUI. + //Execution-time Perf logs are done by either another thread's PerfLogger + //or a reset PerfLogger. + PerfLogger perfLogger = SessionState.getPerfLogger(); + queryDisplay.setPerfLogStarts(QueryDisplay.Phase.COMPILATION, perfLogger.getStartTimes()); + queryDisplay.setPerfLogEnds(QueryDisplay.Phase.COMPILATION, perfLogger.getEndTimes()); return ret; } @@ -1343,6 +1361,8 @@ public class Driver implements CommandProcessor { } perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.DRIVER_RUN); + queryDisplay.setPerfLogStarts(QueryDisplay.Phase.EXECUTION, perfLogger.getStartTimes()); + queryDisplay.setPerfLogEnds(QueryDisplay.Phase.EXECUTION, perfLogger.getEndTimes()); // Take all the driver run hooks and post-execute them. try { @@ -1414,6 +1434,7 @@ public class Driver implements CommandProcessor { } private CommandProcessorResponse createProcessorResponse(int ret) { + queryDisplay.setErrorMessage(errorMessage); return new CommandProcessorResponse(ret, errorMessage, SQLState, downstreamError); } @@ -1544,6 +1565,7 @@ public class Driver implements CommandProcessor { // Launch upto maxthreads tasks Task<? extends Serializable> task; while ((task = driverCxt.getRunnable(maxthreads)) != null) { + queryDisplay.addTask(task); TaskRunner runner = launchTask(task, queryId, noName, jobname, jobs, driverCxt); if (!runner.isRunning()) { break; @@ -1556,6 +1578,7 @@ public class Driver implements CommandProcessor { continue; } hookContext.addCompleteTask(tskRun); + queryDisplay.setTaskCompleted(tskRun.getTask().getId(), tskRun.getTaskResult()); Task<? extends Serializable> tsk = tskRun.getTask(); TaskResult result = tskRun.getTaskResult(); @@ -1694,9 +1717,11 @@ public class Driver implements CommandProcessor { if (noName) { conf.set(MRJobConfig.JOB_NAME, ""); } - dumpMetaCallTimingWithoutEx("execution"); double duration = perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.DRIVER_EXECUTE)/1000.00; + ImmutableMap<String, Long> executionHMSTimings = dumpMetaCallTimingWithoutEx("execution"); + queryDisplay.setHmsTimings(QueryDisplay.Phase.EXECUTION, executionHMSTimings); + Map<String, MapRedStats> stats = SessionState.get().getMapRedStats(); if (stats != null && !stats.isEmpty()) { long totalCpu = 0; @@ -1952,8 +1977,8 @@ public class Driver implements CommandProcessor { } - public String getQueryString() { - return savedQueryString == null ? "Unknown" : savedQueryString; + public QueryDisplay getQueryDisplay() { + return queryDisplay; } /** @@ -1963,5 +1988,4 @@ public class Driver implements CommandProcessor { public void setOperationId(String opId) { this.operationId = opId; } - } http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/ql/src/java/org/apache/hadoop/hive/ql/QueryDisplay.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/QueryDisplay.java b/ql/src/java/org/apache/hadoop/hive/ql/QueryDisplay.java new file mode 100644 index 0000000..c87c825 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/QueryDisplay.java @@ -0,0 +1,253 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql; + +import com.google.common.collect.ImmutableMap; +import org.apache.hadoop.hive.ql.exec.Task; +import org.apache.hadoop.hive.ql.exec.TaskResult; +import org.apache.hadoop.hive.ql.plan.api.StageType; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Some limited query information to save for WebUI. + * + * The class is synchronized, as WebUI may access information about a running query. + */ +public class QueryDisplay { + + // Member variables + private String queryStr; + private String explainPlan; + private String errorMessage; + private String queryId; + + private final Map<Phase, Map<String, Long>> hmsTimingMap = new HashMap(); + private final Map<Phase, Map<String, Long>> perfLogStartMap = new HashMap(); + private final Map<Phase, Map<String, Long>> perfLogEndMap = new HashMap(); + + private final LinkedHashMap<String, TaskInfo> tasks = new LinkedHashMap<String, TaskInfo>(); + + //Inner classes + public static enum Phase { + COMPILATION, + EXECUTION, + } + + public static class TaskInfo { + private Integer returnVal; //if set, determines that task is complete. + private String errorMsg; + private long endTime; + + final long beginTime; + final String taskId; + final StageType taskType; + final String name; + final boolean requireLock; + final boolean retryIfFail; + + public TaskInfo (Task task) { + beginTime = System.currentTimeMillis(); + taskId = task.getId(); + taskType = task.getType(); + name = task.getName(); + requireLock = task.requireLock(); + retryIfFail = task.ifRetryCmdWhenFail(); + } + + public synchronized String getStatus() { + if (returnVal == null) { + return "Running"; + } else if (returnVal == 0) { + return "Success, ReturnVal 0"; + } else { + return "Failure, ReturnVal " + String.valueOf(returnVal); + } + } + + public synchronized long getElapsedTime() { + if (endTime == 0) { + return System.currentTimeMillis() - beginTime; + } else { + return endTime - beginTime; + } + } + + public synchronized String getErrorMsg() { + return errorMsg; + } + + public synchronized long getEndTime() { + return endTime; + } + + //Following methods do not need to be synchronized, because they are final fields. + public long getBeginTime() { + return beginTime; + } + + public String getTaskId() { + return taskId; + } + + public StageType getTaskType() { + return taskType; + } + + public String getName() { + return name; + } + + public boolean isRequireLock() { + return requireLock; + } + + public boolean isRetryIfFail() { + return retryIfFail; + } + } + + public synchronized void addTask(Task task) { + tasks.put(task.getId(), new TaskInfo(task)); + } + + public synchronized void setTaskCompleted(String taskId, TaskResult result) { + TaskInfo taskInfo = tasks.get(taskId); + if (taskInfo != null) { + taskInfo.returnVal = result.getExitVal(); + if (result.getTaskError() != null) { + taskInfo.errorMsg = result.getTaskError().toString(); + } + taskInfo.endTime = System.currentTimeMillis(); + } + } + + public synchronized List<TaskInfo> getTaskInfos() { + List<TaskInfo> taskInfos = new ArrayList<TaskInfo>(); + taskInfos.addAll(tasks.values()); + return taskInfos; + } + + public synchronized void setQueryStr(String queryStr) { + this.queryStr = queryStr; + } + + public synchronized String getQueryString() { + return returnStringOrUnknown(queryStr); + } + + public synchronized String getExplainPlan() { + return returnStringOrUnknown(explainPlan); + } + + public synchronized void setExplainPlan(String explainPlan) { + this.explainPlan = explainPlan; + } + + /** + * @param phase phase of query + * @return map of HMS Client method-calls and duration in miliseconds, during given phase. + */ + public synchronized Map<String, Long> getHmsTimings(Phase phase) { + return hmsTimingMap.get(phase); + } + + /** + * @param phase phase of query + * @param hmsTimings map of HMS Client method-calls and duration in miliseconds, during given phase. + */ + public synchronized void setHmsTimings(Phase phase, ImmutableMap<String, Long> hmsTimings) { + hmsTimingMap.put(phase, hmsTimings); + } + + /** + * @param phase phase of query + * @return map of PerfLogger call-trace name and start time in miliseconds, during given phase. + */ + public synchronized Map<String, Long> getPerfLogStarts(Phase phase) { + return perfLogStartMap.get(phase); + } + + /** + * @param phase phase of query + * @param perfLogStarts map of PerfLogger call-trace name and start time in miliseconds, during given phase. + */ + public synchronized void setPerfLogStarts(Phase phase, ImmutableMap<String, Long> perfLogStarts) { + perfLogStartMap.put(phase, perfLogStarts); + } + + /** + * @param phase phase of query + * @return map of PerfLogger call-trace name and end time in miliseconds, during given phase. + */ + public synchronized Map<String, Long> getPerfLogEnds(Phase phase) { + return perfLogEndMap.get(phase); + } + + /** + * @param phase phase of query + * @param perfLogEnds map of PerfLogger call-trace name and end time in miliseconds, during given phase. + */ + public synchronized void setPerfLogEnds(Phase phase, ImmutableMap<String, Long> perfLogEnds) { + perfLogEndMap.put(phase, perfLogEnds); + } + + /** + * @param phase phase of query + * @return map of PerfLogger call-trace name and duration in miliseconds, during given phase. + */ + public synchronized Map<String, Long> getPerfLogTimes(Phase phase) { + Map<String, Long> times = new HashMap<>(); + Map<String, Long> startTimes = perfLogStartMap.get(phase); + Map<String, Long> endTimes = perfLogEndMap.get(phase); + if (endTimes != null && startTimes != null) { + for (String timeKey : endTimes.keySet()) { + Long endTime = endTimes.get(timeKey); + Long startTime = startTimes.get(timeKey); + if (startTime != null) { + times.put(timeKey, endTime - startTime); + } + } + } + return times; + } + + public synchronized String getErrorMessage() { + return errorMessage; + } + + public synchronized void setErrorMessage(String errorMessage) { + this.errorMessage = errorMessage; + } + + public synchronized String getQueryId() { + return returnStringOrUnknown(queryId); + } + + public synchronized void setQueryId(String queryId) { + this.queryId = queryId; + } + + private String returnStringOrUnknown(String s) { + return s == null ? "UNKNOWN" : s; + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 0bab769..4e574f7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -44,6 +44,7 @@ import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; +import com.google.common.collect.ImmutableMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -3400,7 +3401,7 @@ private void constructOneLBLocationMap(FileStatus fSta, metaCallTimeMap.clear(); } - public void dumpAndClearMetaCallTiming(String phase) { + public ImmutableMap<String, Long> dumpAndClearMetaCallTiming(String phase) { boolean phaseInfoLogged = false; if (LOG.isDebugEnabled()) { phaseInfoLogged = logDumpPhase(phase); @@ -3420,7 +3421,10 @@ private void constructOneLBLocationMap(FileStatus fSta, } } } + + ImmutableMap<String, Long> result = ImmutableMap.copyOf(metaCallTimeMap); metaCallTimeMap.clear(); + return result; } private boolean logDumpPhase(String phase) { http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/service/pom.xml ---------------------------------------------------------------------- diff --git a/service/pom.xml b/service/pom.xml index b2e3a84..e3f61d0 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -128,7 +128,13 @@ <version>${hadoop.version}</version> <optional>true</optional> </dependency> - <!-- intra-project --> + <dependency> + <groupId>org.jamon</groupId> + <artifactId>jamon-runtime</artifactId> + <version>${jamon-runtime.version}</version> + </dependency> + + <!-- intra-project --> <dependency> <groupId>org.apache.hive</groupId> <artifactId>hive-exec</artifactId> @@ -219,7 +225,48 @@ </execution> </executions> </plugin> - <plugin> + <plugin> + <groupId>org.jamon</groupId> + <artifactId>jamon-maven-plugin</artifactId> + <executions> + <execution> + <phase>generate-sources</phase> + <goals> + <goal>translate</goal> + </goals> + <configuration> + <templateSourceDir>src/jamon</templateSourceDir> + <templateOutputDir>target/generated-jamon</templateOutputDir> + </configuration> + </execution> + </executions> + </plugin> + <!-- General plugins --> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-eclipse-plugin</artifactId> + <configuration> + <additionalProjectnatures> + <projectnature>org.jamon.project.jamonnature</projectnature> + </additionalProjectnatures> + <buildcommands> + <buildcommand>org.jamon.project.templateBuilder</buildcommand> + <buildcommand>org.eclipse.jdt.core.javabuilder</buildcommand> + <buildcommand>org.jamon.project.markerUpdater</buildcommand> + </buildcommands> + <additionalConfig> + <file> + <name>.settings/org.jamon.prefs</name> + <content># now + eclipse.preferences.version=1 + templateSourceDir=src/main/jamon + templateOutputDir=target/generated-jamon + </content> + </file> + </additionalConfig> + </configuration> + </plugin> + <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-jar-plugin</artifactId> <executions> http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/service/src/jamon/org/apache/hive/tmpl/QueryProfileTmpl.jamon ---------------------------------------------------------------------- diff --git a/service/src/jamon/org/apache/hive/tmpl/QueryProfileTmpl.jamon b/service/src/jamon/org/apache/hive/tmpl/QueryProfileTmpl.jamon new file mode 100644 index 0000000..c513689 --- /dev/null +++ b/service/src/jamon/org/apache/hive/tmpl/QueryProfileTmpl.jamon @@ -0,0 +1,298 @@ +<%doc> + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +</%doc> +<%args> +SQLOperationDisplay sod; +</%args> +<%import> +java.util.*; +org.apache.hadoop.hive.ql.QueryDisplay; +org.apache.hive.service.cli.operation.SQLOperationDisplay; +</%import> +<!--[if IE]> +<!DOCTYPE html> +<![endif]--> +<?xml version="1.0" encoding="UTF-8" ?> +<html lang="en"> + <head> + <meta charset="utf-8"> + <title>HiveServer2</title> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <meta name="description" content=""> + + <link href="/static/css/bootstrap.min.css" rel="stylesheet"> + <link href="/static/css/bootstrap-theme.min.css" rel="stylesheet"> + <link href="/static/css/hive.css" rel="stylesheet"> + </head> + + <body> + <div class="navbar navbar-fixed-top navbar-default"> + <div class="container"> + <div class="navbar-header"> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse"> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a class="navbar-brand" href="/hiveserver2.jsp"><img src="/static/hive_logo.jpeg" alt="Hive Logo"/></a> + </div> + <div class="collapse navbar-collapse"> + <ul class="nav navbar-nav"> + <li class="active"><a href="/">Home</a></li> + <li><a href="/logs/">Local logs</a></li> + <li><a href="/jmx">Metrics Dump</a></li> + <li><a href="/conf">Hive Configuration</a></li> + <li><a href="/stacks">Stack Trace</a></li> + </ul> + </div><!--/.nav-collapse --> + </div> + </div> + </div> + + + <%if sod == null %> + <div class="jumbotron"> + <p>Query not found. It may have been deleted, increase <i>hive.server2.webui.max.historic.queries</i> + to retain more historic query information.</p> + </div> + <%else> + + + <div class="container"> + <div class="row inner_header"> + <div class="page-header"> + <h1>Query Information: <% sod.getQueryDisplay() == null ? "Unknown" : sod.getQueryDisplay().getQueryString() %></h1> + </div> + </div> + <div class="row"> + + + <div class="tabbable"> + <ul class="nav nav-pills"> + <li class="active"><a href="#tab_baseProfile" data-toggle="tab">Base Profile</a></li> + <li class=""><a href="#tab_stages" data-toggle="tab">Stages</a></li> + <li class=""><a href="#tab_queryPlan" data-toggle="tab">Query Plan</a></li> + <li class=""><a href="#tab_perfLogging" data-toggle="tab">Performance Logging</a></li> + </ul> + <div class="tab-content" style="padding-bottom: 9px; border-bottom: 1px solid #ddd;"> + <div class="tab-pane active" id="tab_baseProfile"> + <& baseProfile; sod = sod &> + </div> + <div class="tab-pane" id="tab_stages"> + <& stages; sod = sod &> + </div> + <div class="tab-pane" id="tab_queryPlan"> + <& queryPlan; sod = sod &> + </div> + <div class="tab-pane" id="tab_perfLogging"> + <& perfLogging; sod = sod &> + </div> + </div> + </div> + + </%if> + +</div> +</div> +<script src="/static/js/jquery.min.js" type="text/javascript"></script> +<script src="/static/js/bootstrap.min.js" type="text/javascript"></script> +<script src="/static/js/tab.js" type="text/javascript"></script> +</body> +</html> + +<%def baseProfile> +<%args> + SQLOperationDisplay sod; +</%args> + <table class="table table-striped"> + <tr> + <td>User Name</td> + <td><% sod.getUserName() %></td> + </tr> + <tr> + <td>Query String</td> + <td><% sod.getQueryDisplay() == null ? "Unknown" : sod.getQueryDisplay().getQueryString() %></td> + </tr> + <tr> + <td>Query Id</td> + <td><% sod.getQueryDisplay() == null ? "Unknown" : sod.getQueryDisplay().getQueryId() %></td> + </tr> + <tr> + <td>Execution Engine</td> + <td><% sod.getExecutionEngine() %> + </tr> + <tr> + <td>State</td> + <td><% sod.getState() %></td> + </tr> + <tr> + <td>Begin Time</td> + <td><% new Date(sod.getBeginTime()) %></td> + </tr> + <tr> + <td>Elapsed Time (s)</td> + <td><% sod.getElapsedTime()/1000 %></td> + </tr> + <tr> + <td>End Time</td> + <td><% sod.getEndTime() == null ? "In Progress" : new Date(sod.getEndTime()) %></td> + </tr> + <%if sod.getQueryDisplay() != null && sod.getQueryDisplay().getErrorMessage() != null %> + <tr> + <td>Error</td> + <td><% sod.getEndTime() == null ? "In Progress" : new Date(sod.getEndTime()) %></td> + </tr> + </%if> + </table> +</%def> + +<%def stages> +<%args> + SQLOperationDisplay sod; +</%args> + <table id="attributes_table" class="table table-striped"> + <tr> + <th>Stage Id</th> + <th>Status</th> + <th>Begin Time</th> + <th>End Time</th> + <th>Elapsed Time (s)</th> + <th>Requires Lock</th> + <th>Retry If Fail</th> + </tr> + + <%if sod.getQueryDisplay() != null && sod.getQueryDisplay().getTaskInfos() != null %> + <%for QueryDisplay.TaskInfo taskInfo : sod.getQueryDisplay().getTaskInfos() %> + <tr> + <td><% taskInfo.getTaskId() + ":" + taskInfo.getTaskType() %></td> + <td><% taskInfo.getStatus() %></td> + <td><% new Date(taskInfo.getBeginTime()) %> + <td><% taskInfo.getEndTime() == 0 ? "" : new Date(taskInfo.getEndTime()) %></td> + <td><% taskInfo.getElapsedTime()/1000 %> (s) </td> + <td><% taskInfo.isRequireLock() %></td> + <td><% taskInfo.isRetryIfFail() %></td> + </tr> + </%for> + </%if> + </table> +</%def> + + +<%def queryPlan> +<%args> + SQLOperationDisplay sod; +</%args> + <div class="panel panel-default"> + <div class="panel-heading">Explain plan</div> + <div class="panel-body"> + <pre> + <% sod.getQueryDisplay() == null ? "Unknown" : sod.getQueryDisplay().getExplainPlan() %> + </pre> + </div> + </div> +</%def> + + +<%def perfLogging> +<%args> + SQLOperationDisplay sod; +</%args> + <section> + <h3>Compile-time metadata operations</h3> + <table id="attributes_table" class="table table-striped"> + <tr> + <th>Call Name</th> + <th>Time (ms)</th> + </tr> + + <%if sod.getQueryDisplay() != null && sod.getQueryDisplay().getHmsTimings(QueryDisplay.Phase.COMPILATION) != null %> + <%for Map.Entry<String, Long> time : sod.getQueryDisplay().getHmsTimings(QueryDisplay.Phase.COMPILATION).entrySet() %> + <tr> + <td><% time.getKey() %></td> + <td><% time.getValue() %></td> + </tr> + </%for> + </%if> + </table> + </section> + + <section> + <h3>Execution-time metadata operations</h3> + <table id="attributes_table" class="table table-striped"> + <tr> + <th>Call Name</th> + <th>Time (ms)</th> + </tr> + + <%if sod.getQueryDisplay() != null && sod.getQueryDisplay().getHmsTimings(QueryDisplay.Phase.EXECUTION) != null %> + <%for Map.Entry<String, Long> time : sod.getQueryDisplay().getHmsTimings(QueryDisplay.Phase.EXECUTION).entrySet() %> + <tr> + <td><% time.getKey() %></td> + <td><% time.getValue() %></td> + </tr> + </%for> + </%if> + </table> + </section> + + <section> + <h3>Compile-Time Perf-Logger</h3> + <table id="attributes_table" class="table table-striped"> + <tr> + <th>Compile-time Call Name</th> + <th>Time (ms)</th> + </tr> + + <%if sod.getQueryDisplay() != null && sod.getQueryDisplay().getPerfLogTimes(QueryDisplay.Phase.COMPILATION) != null %> + <%for Map.Entry<String, Long> time : sod.getQueryDisplay().getPerfLogTimes(QueryDisplay.Phase.COMPILATION).entrySet() %> + <tr> + <td><% time.getKey() %></td> + <td><% time.getValue() %></td> + </tr> + </%for> + </%if> + </table> + </section> + + <section> + <h3>Execution-Time Perf-Logger</h3> + <table id="attributes_table" class="table table-striped"> + <tr> + <th>Execution-time Call Name</th> + <th>Time (ms)</th> + </tr> + + <%if sod.getQueryDisplay() != null && sod.getQueryDisplay().getPerfLogTimes(QueryDisplay.Phase.EXECUTION) != null %> + <%for Map.Entry<String, Long> time : sod.getQueryDisplay().getPerfLogTimes(QueryDisplay.Phase.EXECUTION).entrySet() %> + <tr> + <td><% time.getKey() %></td> + <td><% time.getValue() %></td> + </tr> + </%for> + </%if> + </table> + </section> +</%def> + +</div> +</div> +<script src="/static/js/jquery.min.js" type="text/javascript"></script> +<script src="/static/js/bootstrap.min.js" type="text/javascript"></script> +<script src="/static/js/tab.js" type="text/javascript"></script> +</body> +</html> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/service/src/java/org/apache/hive/service/cli/operation/Operation.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/cli/operation/Operation.java b/service/src/java/org/apache/hive/service/cli/operation/Operation.java index 0c263cf..8340202 100644 --- a/service/src/java/org/apache/hive/service/cli/operation/Operation.java +++ b/service/src/java/org/apache/hive/service/cli/operation/Operation.java @@ -157,6 +157,7 @@ public abstract class Operation { state.validateTransition(newState); this.state = newState; setMetrics(state); + onNewState(state); this.lastAccessTime = System.currentTimeMillis(); return this.state; } @@ -420,4 +421,7 @@ public abstract class Operation { protected OperationState getState() { return state; } + + protected void onNewState(OperationState state) { + } } http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java b/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java index f1ce6f6..38f27ef 100644 --- a/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java +++ b/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java @@ -22,7 +22,9 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -65,8 +67,10 @@ public class OperationManager extends AbstractService { private final ConcurrentHashMap<OperationHandle, Operation> handleToOperation = new ConcurrentHashMap<OperationHandle, Operation>(); - //for displaying historical queries on WebUI - private Queue<SQLOperationInfo> historicSqlOperations; + //Following fields for displaying queries on WebUI + private Object webuiLock = new Object(); + private SQLOperationDisplayCache historicSqlOperations; + private Map<String, SQLOperationDisplay> liveSqlOperations = new LinkedHashMap<String, SQLOperationDisplay>(); public OperationManager() { super(OperationManager.class.getSimpleName()); @@ -80,9 +84,8 @@ public class OperationManager extends AbstractService { } else { LOG.debug("Operation level logging is turned off"); } - if ((hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_PORT) != 0) && - hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_MAX_HISTORIC_QUERIES) > 0) { - historicSqlOperations = EvictingQueue.create( + if (hiveConf.isWebUiQueryInfoCacheEnabled()) { + historicSqlOperations = new SQLOperationDisplayCache( hiveConf.getIntVar(ConfVars.HIVE_SERVER2_WEBUI_MAX_HISTORIC_QUERIES)); } super.init(hiveConf); @@ -186,7 +189,11 @@ public class OperationManager extends AbstractService { Operation operation = handleToOperation.get(operationHandle); if (operation != null && operation.isTimedOut(System.currentTimeMillis())) { handleToOperation.remove(operationHandle, operation); - cacheOldOperationInfo(operation); + synchronized (webuiLock) { + String opKey = operationHandle.getHandleIdentifier().toString(); + SQLOperationDisplay display = liveSqlOperations.remove(opKey); + historicSqlOperations.put(opKey, display); + } return operation; } return null; @@ -194,11 +201,21 @@ public class OperationManager extends AbstractService { private void addOperation(Operation operation) { handleToOperation.put(operation.getHandle(), operation); + if (operation instanceof SQLOperation) { + synchronized (webuiLock) { + liveSqlOperations.put(operation.getHandle().getHandleIdentifier().toString(), + ((SQLOperation) operation).getSQLOperationDisplay()); + } + } } private Operation removeOperation(OperationHandle opHandle) { Operation result = handleToOperation.remove(opHandle); - cacheOldOperationInfo(result); + synchronized (webuiLock) { + String opKey = opHandle.getHandleIdentifier().toString(); + SQLOperationDisplay display = liveSqlOperations.remove(opKey); + historicSqlOperations.put(opKey, display); + } return result; } @@ -329,35 +346,40 @@ public class OperationManager extends AbstractService { return removed; } - //Cache a number of historical operation info, at max number of - //hive.server2.webui.max.historic.queries. - private void cacheOldOperationInfo(Operation oldOperation) { - if ((getHiveConf().getIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_PORT) != 0) && - getHiveConf().getIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_MAX_HISTORIC_QUERIES) > 0) { - if (oldOperation instanceof SQLOperation) { - SQLOperation query = (SQLOperation) oldOperation; - SQLOperationInfo queryInfo = query.getSQLOperationInfo(); - if (queryInfo != null) { - synchronized (historicSqlOperations) { - historicSqlOperations.add(queryInfo); - } - } - } + /** + * @return displays representing a number of historical SQLOperations, at max number of + * hive.server2.webui.max.historic.queries + */ + public List<SQLOperationDisplay> getHistoricalSQLOperations() { + List<SQLOperationDisplay> result = new LinkedList<>(); + synchronized (webuiLock) { + result.addAll(historicSqlOperations.values()); } + return result; } /** - * @return a number of historical SQLOperation info, at max number of - * hive.server2.webui.max.historic.queries + * @return displays representing live SQLOperations */ - public List<SQLOperationInfo> getHistoricalSQLOpInfo() { - List<SQLOperationInfo> result = new LinkedList<>(); - synchronized (historicSqlOperations) { - Iterator<SQLOperationInfo> opIterator = historicSqlOperations.iterator(); - while (opIterator.hasNext()) { - result.add(opIterator.next()); - } + public List<SQLOperationDisplay> getLiveSqlOperations() { + List<SQLOperationDisplay> result = new LinkedList<>(); + synchronized (webuiLock) { + result.addAll(liveSqlOperations.values()); } return result; } + + /** + * @param handle handle of SQLOperation. + * @return display representing a particular SQLOperation. + */ + public SQLOperationDisplay getSQLOperationDisplay(String handle) { + synchronized (webuiLock) { + SQLOperationDisplay result = liveSqlOperations.get(handle); + if (result != null) { + return result; + } + return historicSqlOperations.get(handle); + } + } } http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java index 01b1d3d..3fbbb70 100644 --- a/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java +++ b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java @@ -34,8 +34,6 @@ import java.util.concurrent.RejectedExecutionException; import org.apache.commons.codec.binary.Base64; import org.apache.commons.lang3.CharEncoding; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveVariableSource; -import org.apache.hadoop.hive.conf.VariableSubstitution; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Schema; import org.apache.hadoop.hive.ql.CommandNeedRetryException; @@ -79,11 +77,20 @@ public class SQLOperation extends ExecuteStatementOperation { private SerDe serde = null; private boolean fetchStarted = false; + //Display for WebUI. + private SQLOperationDisplay sqlOpDisplay; + + public SQLOperation(HiveSession parentSession, String statement, Map<String, String> confOverlay, boolean runInBackground) { // TODO: call setRemoteUser in ExecuteStatementOperation or higher. super(parentSession, statement, confOverlay, runInBackground); setupSessionIO(parentSession.getSessionState()); + try { + sqlOpDisplay = new SQLOperationDisplay(this); + } catch (HiveSQLException e) { + LOG.warn("Error calcluating SQL Operation Display for webui", e); + } } private void setupSessionIO(SessionState sessionState) { @@ -111,6 +118,7 @@ public class SQLOperation extends ExecuteStatementOperation { try { driver = new Driver(sqlOperationConf, getParentSession().getUserName()); + sqlOpDisplay.setQueryDisplay(driver.getQueryDisplay()); // set the operation handle information in Driver, so that thrift API users // can use the operation handle they receive, to lookup query information in @@ -162,10 +170,6 @@ public class SQLOperation extends ExecuteStatementOperation { } } - public String getQueryStr() { - return driver == null ? "Unknown" : driver.getQueryString(); - } - private void runQuery(HiveConf sqlOperationConf) throws HiveSQLException { try { // In Hive server mode, we are not able to retry in the FetchTask @@ -485,18 +489,17 @@ public class SQLOperation extends ExecuteStatementOperation { /** * Get summary information of this SQLOperation for display in WebUI. */ - public SQLOperationInfo getSQLOperationInfo() { - try { - return new SQLOperationInfo( - getParentSession().getUserName(), - driver.getQueryString(), - getConfigForOperation().getVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE), - getState(), - (int) (System.currentTimeMillis() - getBeginTime()) / 1000, - System.currentTimeMillis()); - } catch (HiveSQLException e) { - LOG.warn("Error calcluating SQL Operation Info for webui", e); + public SQLOperationDisplay getSQLOperationDisplay() { + return sqlOpDisplay; + } + + @Override + protected void onNewState(OperationState state) { + if (state == OperationState.CLOSED) { + sqlOpDisplay.closed(); + } else { + //CLOSED state not interesting, state before (FINISHED, ERROR) is. + sqlOpDisplay.updateState(state); } - return null; } } http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/service/src/java/org/apache/hive/service/cli/operation/SQLOperationDisplay.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/cli/operation/SQLOperationDisplay.java b/service/src/java/org/apache/hive/service/cli/operation/SQLOperationDisplay.java new file mode 100644 index 0000000..d2ca1e7 --- /dev/null +++ b/service/src/java/org/apache/hive/service/cli/operation/SQLOperationDisplay.java @@ -0,0 +1,99 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.service.cli.operation; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.QueryDisplay; +import org.apache.hive.service.cli.HiveSQLException; +import org.apache.hive.service.cli.OperationState; + +/** + * Used to display some info in the HS2 WebUI. + * + * The class is synchronized, as WebUI may access information about a running query. + */ +public class SQLOperationDisplay { + public final String userName; + public final String executionEngine; + public final long beginTime; + public final String operationId; + + public Long endTime; + public OperationState state; + public QueryDisplay queryDisplay; + + public SQLOperationDisplay(SQLOperation sqlOperation) throws HiveSQLException { + this.state = sqlOperation.getState(); + this.userName = sqlOperation.getParentSession().getUserName(); + this.executionEngine = sqlOperation.getConfigForOperation().getVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE); + this.beginTime = System.currentTimeMillis(); + this.operationId = sqlOperation.getHandle().getHandleIdentifier().toString(); + } + + public synchronized long getElapsedTime() { + if (isRunning()) { + return System.currentTimeMillis() - beginTime; + } else { + return endTime - beginTime; + } + } + + public synchronized boolean isRunning() { + return endTime == null; + } + + public synchronized QueryDisplay getQueryDisplay() { + return queryDisplay; + } + + public synchronized void setQueryDisplay(QueryDisplay queryDisplay) { + this.queryDisplay = queryDisplay; + } + + public String getUserName() { + return userName; + } + + public String getExecutionEngine() { + return executionEngine; + } + + public synchronized OperationState getState() { + return state; + } + + public long getBeginTime() { + return beginTime; + } + + public synchronized Long getEndTime() { + return endTime; + } + + public synchronized void updateState(OperationState state) { + this.state = state; + } + + public String getOperationId() { + return operationId; + } + + public synchronized void closed() { + this.endTime = System.currentTimeMillis(); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/service/src/java/org/apache/hive/service/cli/operation/SQLOperationDisplayCache.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/cli/operation/SQLOperationDisplayCache.java b/service/src/java/org/apache/hive/service/cli/operation/SQLOperationDisplayCache.java new file mode 100644 index 0000000..4a33d37 --- /dev/null +++ b/service/src/java/org/apache/hive/service/cli/operation/SQLOperationDisplayCache.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.service.cli.operation; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Cache some SQLOperation information for WebUI + */ +public class SQLOperationDisplayCache extends LinkedHashMap<String, SQLOperationDisplay> { + + private final int capacity; + + public SQLOperationDisplayCache(int capacity) { + super(capacity + 1, 1.1f, false); + this.capacity = capacity; + } + + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > capacity; + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/service/src/java/org/apache/hive/service/cli/operation/SQLOperationInfo.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/cli/operation/SQLOperationInfo.java b/service/src/java/org/apache/hive/service/cli/operation/SQLOperationInfo.java deleted file mode 100644 index 179f6dd..0000000 --- a/service/src/java/org/apache/hive/service/cli/operation/SQLOperationInfo.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hive.service.cli.operation; - -import org.apache.hive.service.cli.OperationState; - -/** - * Used to display some info in the HS2 WebUI. - */ -public class SQLOperationInfo { - public String userName; - public String queryStr; - public String executionEngine; - public OperationState endState; //state before CLOSED (one of CANCELLED, FINISHED, ERROR) - public int elapsedTime; - public long endTime; - - public SQLOperationInfo( - String userName, - String queryStr, - String executionEngine, - OperationState endState, - int elapsedTime, - long endTime - ) { - this.userName = userName; - this.queryStr = queryStr; - this.executionEngine = executionEngine; - this.endState = endState; - this.elapsedTime = elapsedTime; - this.endTime = endTime; - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/service/src/java/org/apache/hive/service/server/HiveServer2.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/server/HiveServer2.java b/service/src/java/org/apache/hive/service/server/HiveServer2.java index 958458f..c601614 100644 --- a/service/src/java/org/apache/hive/service/server/HiveServer2.java +++ b/service/src/java/org/apache/hive/service/server/HiveServer2.java @@ -64,6 +64,7 @@ import org.apache.hive.service.cli.CLIService; import org.apache.hive.service.cli.thrift.ThriftBinaryCLIService; import org.apache.hive.service.cli.thrift.ThriftCLIService; import org.apache.hive.service.cli.thrift.ThriftHttpCLIService; +import org.apache.hive.service.servlet.QueryProfileServlet; import org.apache.logging.log4j.util.Strings; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; @@ -180,6 +181,7 @@ public class HiveServer2 extends CompositeService { builder.setUseSPNEGO(true); } webServer = builder.build(); + webServer.addServlet("query_page", "/query_page", QueryProfileServlet.class); } } } catch (IOException ie) { http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/service/src/java/org/apache/hive/service/servlet/QueryProfileServlet.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/servlet/QueryProfileServlet.java b/service/src/java/org/apache/hive/service/servlet/QueryProfileServlet.java new file mode 100644 index 0000000..74a374d --- /dev/null +++ b/service/src/java/org/apache/hive/service/servlet/QueryProfileServlet.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.service.servlet; + +import org.apache.hive.service.cli.operation.OperationManager; +import org.apache.hive.service.cli.operation.SQLOperationDisplay; +import org.apache.hive.service.cli.session.SessionManager; +import org.apache.hive.tmpl.QueryProfileTmpl; + +import javax.servlet.ServletContext; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; + +/** + * Renders a query page + */ +public class QueryProfileServlet extends HttpServlet { + @Override + public void doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + String opId = (String) request.getParameter("operationId"); + ServletContext ctx = getServletContext(); + SessionManager sessionManager = + (SessionManager)ctx.getAttribute("hive.sm"); + OperationManager opManager = sessionManager.getOperationManager(); + SQLOperationDisplay sod = opManager.getSQLOperationDisplay(opId); + + new QueryProfileTmpl().render(response.getWriter(), sod); + } + +} http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp ---------------------------------------------------------------------- diff --git a/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp b/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp index a0b5d2e..8b46550 100644 --- a/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp +++ b/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp @@ -24,7 +24,7 @@ import="org.apache.hive.common.util.HiveVersionInfo" import="org.apache.hive.service.cli.operation.Operation" import="org.apache.hive.service.cli.operation.SQLOperation" - import="org.apache.hive.service.cli.operation.SQLOperationInfo" + import="org.apache.hive.service.cli.operation.SQLOperationDisplay" import="org.apache.hive.service.cli.session.SessionManager" import="org.apache.hive.service.cli.session.HiveSession" import="javax.servlet.ServletContext" @@ -132,29 +132,32 @@ for (HiveSession hiveSession: hiveSessions) { <th>Query</th> <th>Execution Engine</th> <th>State</th> + <th>Begin Time</th> <th>Elapsed Time (s)</th> + <th>Drilldown Link</th> </tr> -<% -int queries = 0; -Collection<Operation> operations = sessionManager.getOperations(); -for (Operation operation: operations) { - if (operation instanceof SQLOperation) { - SQLOperation query = (SQLOperation) operation; - queries++; -%> + <% + int queries = 0; + Collection<SQLOperationDisplay> operations = sessionManager.getOperationManager().getLiveSqlOperations(); + for (SQLOperationDisplay operation : operations) { + queries++; + %> <tr> - <td><%= query.getParentSession().getUserName() %></td> - <td><%= query.getQueryStr() %></td> - <td><%= query.getConfigForOperation().getVar(ConfVars.HIVE_EXECUTION_ENGINE) %> - <td><%= query.getStatus().getState() %></td> - <td><%= (currentTime - query.getBeginTime())/1000 %></td> + <td><%= operation.getUserName() %></td> + <td><%= operation.getQueryDisplay() == null ? "Unknown" : operation.getQueryDisplay().getQueryString() %></td> + <td><%= operation.getExecutionEngine() %> + <td><%= operation.getState() %></td> + <td><%= new Date(operation.getBeginTime()) %></td> + <td><%= operation.getElapsedTime()/1000 %></td> + <% String link = "/query_page?operationId=" + operation.getOperationId(); %> + <td> <a href= <%= link %>>Query Drilldown</a> </td> </tr> + <% } -} %> <tr> - <td colspan="5">Total number of queries: <%= queries %></td> + <td colspan="7">Total number of queries: <%= queries %></td> </tr> </table> </section> @@ -169,33 +172,36 @@ for (Operation operation: operations) { <th>Execution Engine</th> <th>State</th> <th>Elapsed Time (s)</th> - <th>End Time </th> + <th>End Time</th> + <th>Drilldown Link</th> </tr> -<% -queries = 0; -List<SQLOperationInfo> sqlOperations = sessionManager.getOperationManager().getHistoricalSQLOpInfo(); -for (SQLOperationInfo sqlOperation: sqlOperations) { - queries++; -%> + <% + queries = 0; + operations = sessionManager.getOperationManager().getHistoricalSQLOperations(); + for (SQLOperationDisplay operation : operations) { + queries++; + %> <tr> - <td><%= sqlOperation.userName %></td> - <td><%= sqlOperation.queryStr %></td> - <td><%= sqlOperation.executionEngine %></td> - <td><%= sqlOperation.endState %></td> - <td><%= sqlOperation.elapsedTime %></td> - <td><%= new Date(sqlOperation.endTime) %></td> + <td><%= operation.getUserName() %></td> + <td><%= operation.getQueryDisplay() == null ? "Unknown" : operation.getQueryDisplay().getQueryString() %></td> + <td><%= operation.getExecutionEngine() %> + <td><%= operation.getState() %></td> + <td><%= operation.getElapsedTime()/1000 %></td> + <td><%= operation.getEndTime() == null ? "In Progress" : new Date(operation.getEndTime()) %></td> + <% String link = "/query_page?operationId=" + operation.getOperationId(); %> + <td> <a href= <%= link %>>Query Drilldown</a> </td> </tr> -<% -} +<% + } %> <tr> - <td colspan="6">Total number of queries: <%= queries %></td> + <td colspan="8">Total number of queries: <%= queries %></td> </tr> </table> </section> -<% +<% } %>
