Repository: hive Updated Branches: refs/heads/master 41fc1874b -> 3db7227fc
HIVE-12550 : Cache and display last N completed queries in HS2 WebUI (Szehon, reviewed by Aihua Xu and Mohit Sabharwal) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3db7227f Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3db7227f Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3db7227f Branch: refs/heads/master Commit: 3db7227fc7e40ef65d59a8e5d26b9c83bc6fe08c Parents: 41fc187 Author: Szehon Ho <sze...@cloudera.com> Authored: Fri Jan 22 15:37:37 2016 -0500 Committer: Szehon Ho <sze...@cloudera.com> Committed: Fri Jan 29 10:36:34 2016 -0800 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/conf/HiveConf.java | 2 + pom.xml | 2 +- .../java/org/apache/hadoop/hive/ql/Driver.java | 9 ++++ .../hive/service/cli/operation/Operation.java | 12 ++++- .../service/cli/operation/OperationManager.java | 49 +++++++++++++++++++- .../service/cli/operation/SQLOperation.java | 22 ++++++++- .../service/cli/operation/SQLOperationInfo.java | 48 +++++++++++++++++++ .../hive-webapps/hiveserver2/hiveserver2.jsp | 41 +++++++++++++++- 8 files changed, 180 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/3db7227f/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index bfd88f8..a78f78a 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1910,6 +1910,8 @@ public class HiveConf extends Configuration { "HTTP/_h...@example.com", "The HiveServer2 WebUI SPNEGO service principal.\n" + "The special string _HOST will be replaced automatically with \n" + "the value of hive.server2.webui.host or the correct host name."), + HIVE_SERVER2_WEBUI_MAX_HISTORIC_QUERIES("hive.server2.webui.max.historic.queries", 25, + "The maximum number of past queries to show in HiverSever2 WebUI."), // Tez session settings HIVE_SERVER2_TEZ_DEFAULT_QUEUES("hive.server2.tez.default.queues", "", http://git-wip-us.apache.org/repos/asf/hive/blob/3db7227f/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 2d2a3de..802d3d4 100644 --- a/pom.xml +++ b/pom.xml @@ -125,7 +125,7 @@ <commons-dbcp.version>1.4</commons-dbcp.version> <derby.version>10.10.2.0</derby.version> <dropwizard.version>3.1.0</dropwizard.version> - <guava.version>14.0.1</guava.version> + <guava.version>15.0</guava.version> <groovy.version>2.4.4</groovy.version> <hadoop.version>2.6.0</hadoop.version> <hadoop.bin.path>${basedir}/${hive.path.to.root}/testutils/hadoop</hadoop.bin.path> http://git-wip-us.apache.org/repos/asf/hive/blob/3db7227f/ql/src/java/org/apache/hadoop/hive/ql/Driver.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java index 75187cf..4c89812 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -158,6 +158,9 @@ public class Driver implements CommandProcessor { // HS2 operation handle guid string private String operationId; + // For WebUI. Kept alive after queryPlan is freed. + private String savedQueryString; + private boolean checkConcurrency() { boolean supportConcurrency = conf.getBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY); if (!supportConcurrency) { @@ -384,6 +387,7 @@ public class Driver implements CommandProcessor { } catch (Exception e) { LOG.warn("WARNING! Query command could not be redacted." + e); } + this.savedQueryString = queryStr; //holder for parent command type/string when executing reentrant queries QueryState queryState = new QueryState(); @@ -1943,6 +1947,11 @@ public class Driver implements CommandProcessor { return errorMessage; } + + public String getQueryString() { + return savedQueryString == null ? "Unknown" : savedQueryString; + } + /** * Set the HS2 operation handle's guid string * @param opId base64 encoded guid string http://git-wip-us.apache.org/repos/asf/hive/blob/3db7227f/service/src/java/org/apache/hive/service/cli/operation/Operation.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/cli/operation/Operation.java b/service/src/java/org/apache/hive/service/cli/operation/Operation.java index d6ac16d..0c263cf 100644 --- a/service/src/java/org/apache/hive/service/cli/operation/Operation.java +++ b/service/src/java/org/apache/hive/service/cli/operation/Operation.java @@ -75,6 +75,7 @@ public abstract class Operation { private long operationTimeout; private volatile long lastAccessTime; + private final long beginTime; protected static final EnumSet<FetchOrientation> DEFAULT_FETCH_ORIENTATION_SET = EnumSet.of(FetchOrientation.FETCH_NEXT,FetchOrientation.FETCH_FIRST); @@ -92,7 +93,8 @@ public abstract class Operation { } this.runAsync = runInBackground; this.opHandle = new OperationHandle(opType, parentSession.getProtocolVersion()); - lastAccessTime = System.currentTimeMillis(); + beginTime = System.currentTimeMillis(); + lastAccessTime = beginTime; operationTimeout = HiveConf.getTimeVar(parentSession.getHiveConf(), HiveConf.ConfVars.HIVE_SERVER2_IDLE_OPERATION_TIMEOUT, TimeUnit.MILLISECONDS); setMetrics(state); @@ -410,4 +412,12 @@ public abstract class Operation { } } } + + public long getBeginTime() { + return beginTime; + } + + protected OperationState getState() { + return state; + } } http://git-wip-us.apache.org/repos/asf/hive/blob/3db7227f/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java b/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java index 92135cd..f1ce6f6 100644 --- a/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java +++ b/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java @@ -22,10 +22,14 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Queue; import java.util.concurrent.ConcurrentHashMap; +import com.google.common.collect.EvictingQueue; import org.apache.hadoop.hive.common.metrics.common.Metrics; import org.apache.hadoop.hive.common.metrics.common.MetricsConstant; import org.apache.hadoop.hive.common.metrics.common.MetricsFactory; @@ -61,6 +65,9 @@ public class OperationManager extends AbstractService { private final ConcurrentHashMap<OperationHandle, Operation> handleToOperation = new ConcurrentHashMap<OperationHandle, Operation>(); + //for displaying historical queries on WebUI + private Queue<SQLOperationInfo> historicSqlOperations; + public OperationManager() { super(OperationManager.class.getSimpleName()); } @@ -73,6 +80,11 @@ public class OperationManager extends AbstractService { } else { LOG.debug("Operation level logging is turned off"); } + if ((hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_PORT) != 0) && + hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_MAX_HISTORIC_QUERIES) > 0) { + historicSqlOperations = EvictingQueue.create( + hiveConf.getIntVar(ConfVars.HIVE_SERVER2_WEBUI_MAX_HISTORIC_QUERIES)); + } super.init(hiveConf); } @@ -174,6 +186,7 @@ public class OperationManager extends AbstractService { Operation operation = handleToOperation.get(operationHandle); if (operation != null && operation.isTimedOut(System.currentTimeMillis())) { handleToOperation.remove(operationHandle, operation); + cacheOldOperationInfo(operation); return operation; } return null; @@ -184,7 +197,9 @@ public class OperationManager extends AbstractService { } private Operation removeOperation(OperationHandle opHandle) { - return handleToOperation.remove(opHandle); + Operation result = handleToOperation.remove(opHandle); + cacheOldOperationInfo(result); + return result; } public OperationStatus getOperationStatus(OperationHandle opHandle) @@ -313,4 +328,36 @@ public class OperationManager extends AbstractService { } return removed; } + + //Cache a number of historical operation info, at max number of + //hive.server2.webui.max.historic.queries. + private void cacheOldOperationInfo(Operation oldOperation) { + if ((getHiveConf().getIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_PORT) != 0) && + getHiveConf().getIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_MAX_HISTORIC_QUERIES) > 0) { + if (oldOperation instanceof SQLOperation) { + SQLOperation query = (SQLOperation) oldOperation; + SQLOperationInfo queryInfo = query.getSQLOperationInfo(); + if (queryInfo != null) { + synchronized (historicSqlOperations) { + historicSqlOperations.add(queryInfo); + } + } + } + } + } + + /** + * @return a number of historical SQLOperation info, at max number of + * hive.server2.webui.max.historic.queries + */ + public List<SQLOperationInfo> getHistoricalSQLOpInfo() { + List<SQLOperationInfo> result = new LinkedList<>(); + synchronized (historicSqlOperations) { + Iterator<SQLOperationInfo> opIterator = historicSqlOperations.iterator(); + while (opIterator.hasNext()) { + result.add(opIterator.next()); + } + } + return result; + } } http://git-wip-us.apache.org/repos/asf/hive/blob/3db7227f/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java index c8a69b9..01b1d3d 100644 --- a/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java +++ b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java @@ -163,7 +163,7 @@ public class SQLOperation extends ExecuteStatementOperation { } public String getQueryStr() { - return driver == null || driver.getPlan() == null ? "Unknown" : driver.getPlan().getQueryStr(); + return driver == null ? "Unknown" : driver.getQueryString(); } private void runQuery(HiveConf sqlOperationConf) throws HiveSQLException { @@ -199,6 +199,7 @@ public class SQLOperation extends ExecuteStatementOperation { public void runInternal() throws HiveSQLException { setState(OperationState.PENDING); final HiveConf opConfig = getConfigForOperation(); + prepare(opConfig); if (!shouldRunAsync()) { runQuery(opConfig); @@ -304,6 +305,7 @@ public class SQLOperation extends ExecuteStatementOperation { backgroundHandle.cancel(true); } } + if (driver != null) { driver.close(); driver.destroy(); @@ -479,4 +481,22 @@ public class SQLOperation extends ExecuteStatementOperation { } return sqlOperationConf; } + + /** + * Get summary information of this SQLOperation for display in WebUI. + */ + public SQLOperationInfo getSQLOperationInfo() { + try { + return new SQLOperationInfo( + getParentSession().getUserName(), + driver.getQueryString(), + getConfigForOperation().getVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE), + getState(), + (int) (System.currentTimeMillis() - getBeginTime()) / 1000, + System.currentTimeMillis()); + } catch (HiveSQLException e) { + LOG.warn("Error calcluating SQL Operation Info for webui", e); + } + return null; + } } http://git-wip-us.apache.org/repos/asf/hive/blob/3db7227f/service/src/java/org/apache/hive/service/cli/operation/SQLOperationInfo.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/cli/operation/SQLOperationInfo.java b/service/src/java/org/apache/hive/service/cli/operation/SQLOperationInfo.java new file mode 100644 index 0000000..179f6dd --- /dev/null +++ b/service/src/java/org/apache/hive/service/cli/operation/SQLOperationInfo.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.service.cli.operation; + +import org.apache.hive.service.cli.OperationState; + +/** + * Used to display some info in the HS2 WebUI. + */ +public class SQLOperationInfo { + public String userName; + public String queryStr; + public String executionEngine; + public OperationState endState; //state before CLOSED (one of CANCELLED, FINISHED, ERROR) + public int elapsedTime; + public long endTime; + + public SQLOperationInfo( + String userName, + String queryStr, + String executionEngine, + OperationState endState, + int elapsedTime, + long endTime + ) { + this.userName = userName; + this.queryStr = queryStr; + this.executionEngine = executionEngine; + this.endState = endState; + this.elapsedTime = elapsedTime; + this.endTime = endTime; + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/3db7227f/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp ---------------------------------------------------------------------- diff --git a/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp b/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp index a91b008..a0b5d2e 100644 --- a/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp +++ b/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp @@ -24,11 +24,13 @@ import="org.apache.hive.common.util.HiveVersionInfo" import="org.apache.hive.service.cli.operation.Operation" import="org.apache.hive.service.cli.operation.SQLOperation" + import="org.apache.hive.service.cli.operation.SQLOperationInfo" import="org.apache.hive.service.cli.session.SessionManager" import="org.apache.hive.service.cli.session.HiveSession" import="javax.servlet.ServletContext" import="java.util.Collection" import="java.util.Date" + import="java.util.List" %> <% @@ -145,7 +147,7 @@ for (Operation operation: operations) { <td><%= query.getQueryStr() %></td> <td><%= query.getConfigForOperation().getVar(ConfVars.HIVE_EXECUTION_ENGINE) %> <td><%= query.getStatus().getState() %></td> - <td><%= (currentTime - query.getLastAccessTime())/1000 %></td> + <td><%= (currentTime - query.getBeginTime())/1000 %></td> </tr> <% } @@ -156,6 +158,43 @@ for (Operation operation: operations) { </tr> </table> </section> + + +<section> +<h2>Last Max <%= conf.get(ConfVars.HIVE_SERVER2_WEBUI_MAX_HISTORIC_QUERIES.varname) %> Completed Queries</h2> +<table id="attributes_table" class="table table-striped"> + <tr> + <th>User Name</th> + <th>Query</th> + <th>Execution Engine</th> + <th>State</th> + <th>Elapsed Time (s)</th> + <th>End Time </th> + </tr> +<% +queries = 0; +List<SQLOperationInfo> sqlOperations = sessionManager.getOperationManager().getHistoricalSQLOpInfo(); +for (SQLOperationInfo sqlOperation: sqlOperations) { + queries++; +%> + <tr> + <td><%= sqlOperation.userName %></td> + <td><%= sqlOperation.queryStr %></td> + <td><%= sqlOperation.executionEngine %></td> + <td><%= sqlOperation.endState %></td> + <td><%= sqlOperation.elapsedTime %></td> + <td><%= new Date(sqlOperation.endTime) %></td> + </tr> +<% +} + +%> +<tr> + <td colspan="6">Total number of queries: <%= queries %></td> +</tr> +</table> +</section> + <% } %>