Updated Branches: refs/heads/blur-console-v2 ac584ad1c -> f102d62a7
Got the queries dashboard pieces running Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/f102d62a Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/f102d62a Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/f102d62a Branch: refs/heads/blur-console-v2 Commit: f102d62a74723c0fba8b7024ebebebc4fb19cee6 Parents: ac584ad Author: Chris Rohr <[email protected]> Authored: Fri Nov 15 22:57:17 2013 -0500 Committer: Chris Rohr <[email protected]> Committed: Fri Nov 15 22:57:17 2013 -0500 ---------------------------------------------------------------------- .../org/apache/blur/console/JettyServer.java | 2 + .../blur/console/servlets/QueriesServlet.java | 85 ++++++++++++ .../blur/console/servlets/TablesServlet.java | 5 - .../org/apache/blur/console/util/QueryUtil.java | 112 ++++++++++++++-- .../org/apache/blur/console/util/TableUtil.java | 2 +- .../apache/blur/console/webapp/js/dashboard.js | 128 +++++++++++++++---- .../apache/blur/console/webapp/js/directives.js | 17 ++- .../console/webapp/partials/dashboard.tpl.html | 8 +- 8 files changed, 309 insertions(+), 50 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/f102d62a/contrib/blur-console/src/main/java/org/apache/blur/console/JettyServer.java ---------------------------------------------------------------------- diff --git a/contrib/blur-console/src/main/java/org/apache/blur/console/JettyServer.java b/contrib/blur-console/src/main/java/org/apache/blur/console/JettyServer.java index c265a35..de7381b 100644 --- a/contrib/blur-console/src/main/java/org/apache/blur/console/JettyServer.java +++ b/contrib/blur-console/src/main/java/org/apache/blur/console/JettyServer.java @@ -20,6 +20,7 @@ package org.apache.blur.console; import java.net.URL; import org.apache.blur.console.servlets.DashboardServlet; +import org.apache.blur.console.servlets.QueriesServlet; import org.apache.blur.console.servlets.TablesServlet; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -65,6 +66,7 @@ public class JettyServer { final Context context = new Context(server, "/service", Context.SESSIONS); context.addServlet(new ServletHolder(new DashboardServlet()), "/dashboard/*"); context.addServlet(new ServletHolder(new TablesServlet()), "/tables/*"); + context.addServlet(new ServletHolder(new QueriesServlet()), "/queries/*"); try { server.start(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/f102d62a/contrib/blur-console/src/main/java/org/apache/blur/console/servlets/QueriesServlet.java ---------------------------------------------------------------------- diff --git a/contrib/blur-console/src/main/java/org/apache/blur/console/servlets/QueriesServlet.java b/contrib/blur-console/src/main/java/org/apache/blur/console/servlets/QueriesServlet.java new file mode 100644 index 0000000..533a3fe --- /dev/null +++ b/contrib/blur-console/src/main/java/org/apache/blur/console/servlets/QueriesServlet.java @@ -0,0 +1,85 @@ +package org.apache.blur.console.servlets; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.blur.console.util.HttpUtil; +import org.apache.blur.console.util.NodeUtil; +import org.apache.blur.console.util.QueryUtil; +import org.apache.blur.console.util.TableUtil; +import org.apache.blur.thirdparty.thrift_0_9_0.TException; +import org.apache.blur.thrift.generated.BlurException; +import org.apache.commons.io.IOUtils; +import org.codehaus.jackson.map.ObjectMapper; + +public class QueriesServlet extends HttpServlet { + private static final long serialVersionUID = -5725846390100596115L; +// private static Pattern tableTermsPattern = Pattern.compile("/(.*)/(.*)/terms"); + + @Override + protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { + String path = request.getPathInfo(); + Matcher m; + if ("/status".equalsIgnoreCase(path)) { + sendQueryStatus(response); + } +// else if ((m = tableTermsPattern.matcher(path)).matches()) { +//// sendTerms(response, m.group(1), m.group(2)); +// } +// else if ("/query/status".equalsIgnoreCase(path)) { +// sendQueryStatus(response); +// } + else { + response.setStatus(404); + IOUtils.write("Route [" + path + "] doesn't exist", response.getOutputStream()); + } + } + + private void sendError(HttpServletResponse response, Exception e) throws IOException { + String body = e.getMessage(); + response.setContentType("application/json"); + response.setContentLength(body.getBytes().length); + response.setStatus(500); + IOUtils.write(body, response.getOutputStream()); + } + + private void sendQueryStatus(HttpServletResponse response) throws IOException { + Map<String, Object> queries = new HashMap<String, Object>(); + try { + queries = QueryUtil.getQueryStatus(); + } catch (IOException e) { + throw new IOException(e); + } catch (Exception e) { + sendError(response, e); + return; + } + + HttpUtil.sendResponse(response, new ObjectMapper().writeValueAsString(queries), HttpUtil.JSON); + } +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/f102d62a/contrib/blur-console/src/main/java/org/apache/blur/console/servlets/TablesServlet.java ---------------------------------------------------------------------- diff --git a/contrib/blur-console/src/main/java/org/apache/blur/console/servlets/TablesServlet.java b/contrib/blur-console/src/main/java/org/apache/blur/console/servlets/TablesServlet.java index 7f27f1f..a71e0b8 100644 --- a/contrib/blur-console/src/main/java/org/apache/blur/console/servlets/TablesServlet.java +++ b/contrib/blur-console/src/main/java/org/apache/blur/console/servlets/TablesServlet.java @@ -19,7 +19,6 @@ package org.apache.blur.console.servlets; import java.io.IOException; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -30,11 +29,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.apache.blur.console.util.HttpUtil; -import org.apache.blur.console.util.NodeUtil; -import org.apache.blur.console.util.QueryUtil; import org.apache.blur.console.util.TableUtil; -import org.apache.blur.thirdparty.thrift_0_9_0.TException; -import org.apache.blur.thrift.generated.BlurException; import org.apache.commons.io.IOUtils; import org.codehaus.jackson.map.ObjectMapper; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/f102d62a/contrib/blur-console/src/main/java/org/apache/blur/console/util/QueryUtil.java ---------------------------------------------------------------------- diff --git a/contrib/blur-console/src/main/java/org/apache/blur/console/util/QueryUtil.java b/contrib/blur-console/src/main/java/org/apache/blur/console/util/QueryUtil.java index 7c4a46e..cc9c325 100644 --- a/contrib/blur-console/src/main/java/org/apache/blur/console/util/QueryUtil.java +++ b/contrib/blur-console/src/main/java/org/apache/blur/console/util/QueryUtil.java @@ -18,44 +18,130 @@ package org.apache.blur.console.util; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Calendar; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.TreeMap; import org.apache.blur.thirdparty.thrift_0_9_0.TException; import org.apache.blur.thrift.BlurClient; import org.apache.blur.thrift.generated.Blur.Iface; import org.apache.blur.thrift.generated.BlurException; import org.apache.blur.thrift.generated.BlurQueryStatus; +import org.apache.blur.thrift.generated.CpuTime; +import org.apache.blur.thrift.generated.QueryState; public class QueryUtil { + /** + * { + * clusters: ['prodA', 'prodB'], + * counts: { + * 'prodA':{ + * timeToTheMinute: count + * } + * }, + * performance: { + * 'prodA':{ + * timeToTheMinute: avgCpuTime + * } + * }, + * slowQueries: count + * } + */ public static Map<String, Object> getQueryStatus() throws IOException, BlurException, TException { Iface client = BlurClient.getClient(Config.getConnectionString()); - Map<String, Integer> clusterQueryCount = new HashMap<String, Integer>(); + Map<String, Object> queryStatus = new HashMap<String, Object>(); + List<String> clusters = client.shardClusterList(); + queryStatus.put("clusters", clusters); + + Integer slowQueries = 0; + + Calendar slowThreshold = Calendar.getInstance(); + slowThreshold.add(Calendar.MINUTE, -1); + + Map<String, Map<Long, Long>> queryCounts = new HashMap<String, Map<Long, Long>>(); + Map<String, Map<Long, Long>> queryPerformances = new HashMap<String, Map<Long, Long>>(); for (String cluster : clusters) { List<String> tables = client.tableListByCluster(cluster); - int queryCount = 0; + + Map<Long, List<Long>> queryPerfs = new TreeMap<Long, List<Long>>(); + Map<Long, Long> clusterQueries = new TreeMap<Long,Long>(); for (String table : tables) { - List<String> queryIds = client.queryStatusIdList(table); - queryCount += queryIds.size(); - System.out.println(queryCount); - - for (String queryId : queryIds) { - BlurQueryStatus status = client.queryStatusById(table, queryId); - System.out.println(status.getCpuTimes()); - System.out.println(status.getQuery().getStartTime()); + if (client.describe(table).isEnabled()) { + List<String> queryIds = client.queryStatusIdList(table); + + if (queryIds.isEmpty()) { + Calendar cal = Calendar.getInstance(); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + clusterQueries.put(cal.getTimeInMillis(), 0L); + queryPerfs.put(cal.getTimeInMillis(), new ArrayList<Long>(Arrays.asList(new Long[]{0L}))); + } else { + for (String queryId : queryIds) { + BlurQueryStatus status = client.queryStatusById(table, queryId); + + Calendar cal = Calendar.getInstance(); + cal.setTimeInMillis(status.getQuery().getStartTime()); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + + if (QueryState.RUNNING.equals(status.getState()) && cal.getTimeInMillis() < slowThreshold.getTimeInMillis()) { + slowQueries++; + } + + incrementQueryCount(clusterQueries, cal); + collectQueryPerformance(queryPerfs, cal, status); + } + } } } - clusterQueryCount.put(cluster, queryCount); + Map<Long, Long> clusterQueryPerfs = new TreeMap<Long,Long>(); + for (Map.Entry<Long, List<Long>> perfEntry : queryPerfs.entrySet()) { + long sum = 0; + for (Long perf : perfEntry.getValue()) { + sum += perf; + } + clusterQueryPerfs.put(perfEntry.getKey(), (sum/perfEntry.getValue().size())); + } + + queryCounts.put(cluster, clusterQueries); + queryPerformances.put(cluster, clusterQueryPerfs); } - System.out.println(clusterQueryCount); + queryStatus.put("counts", queryCounts); + queryStatus.put("performance", queryPerformances); + queryStatus.put("slowQueries", slowQueries); - return null; + return queryStatus; + } + + private static void incrementQueryCount(Map<Long, Long> queries, Calendar cal) { + Long previousCount = queries.get(cal.getTimeInMillis()); + if (previousCount == null) { + previousCount = 0L; + } + + queries.put(cal.getTimeInMillis(), previousCount + 1); + } + + private static void collectQueryPerformance(Map<Long, List<Long>> queries, Calendar cal, BlurQueryStatus status) { + List<Long> perfs = queries.get(cal.getTimeInMillis()); + if (perfs == null) { + perfs = new ArrayList<Long>(); + queries.put(cal.getTimeInMillis(), perfs); + } + + Map<String, CpuTime> cpuTimes = status.getCpuTimes(); + for (CpuTime cpu : cpuTimes.values()) { + perfs.add(cpu.getRealTime()); + } } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/f102d62a/contrib/blur-console/src/main/java/org/apache/blur/console/util/TableUtil.java ---------------------------------------------------------------------- diff --git a/contrib/blur-console/src/main/java/org/apache/blur/console/util/TableUtil.java b/contrib/blur-console/src/main/java/org/apache/blur/console/util/TableUtil.java index cb23c4c..b17e1f3 100644 --- a/contrib/blur-console/src/main/java/org/apache/blur/console/util/TableUtil.java +++ b/contrib/blur-console/src/main/java/org/apache/blur/console/util/TableUtil.java @@ -119,12 +119,12 @@ public class TableUtil { Map<String, Object> tableInfo = new HashMap<String, Object>(); TableDescriptor descriptor = client.describe(table); - TableStats stats = client.tableStats(table); tableInfo.put("name", table); tableInfo.put("enabled", descriptor.isEnabled()); if (descriptor.isEnabled()) { + TableStats stats = client.tableStats(table); tableInfo.put("rowCount", stats.getRowCount()); tableInfo.put("recordCount", stats.getRecordCount()); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/f102d62a/contrib/blur-console/src/main/resources/org/apache/blur/console/webapp/js/dashboard.js ---------------------------------------------------------------------- diff --git a/contrib/blur-console/src/main/resources/org/apache/blur/console/webapp/js/dashboard.js b/contrib/blur-console/src/main/resources/org/apache/blur/console/webapp/js/dashboard.js index eb7f0ec..66876cd 100644 --- a/contrib/blur-console/src/main/resources/org/apache/blur/console/webapp/js/dashboard.js +++ b/contrib/blur-console/src/main/resources/org/apache/blur/console/webapp/js/dashboard.js @@ -22,9 +22,15 @@ blurApp.controller('DashboardCtrl', function($scope, $http, $timeout) { $scope.nodeData = {}; $scope.tableData = {}; - $scope.queryData = {}; + $scope.queryData = {counters:[], performance:[]}; $scope.hasslowwarnings = false; + var now = new Date(); + now.setMilliseconds(0); + now.setSeconds(0); + var nowTime = now.getTime(); + + (function node_tick() { $http.get('/service/dashboard/node/status').success(function (data) { $scope.nodeData = data; @@ -45,28 +51,100 @@ blurApp.controller('DashboardCtrl', function($scope, $http, $timeout) { }); })(); - // (function query_tick() { - // $http.get('/service/dashboard/query/status').success(function (data) { - // $scope.queryData.errMsg = data.errMsg; - - // var chartData = []; - // angular.forEach(data.clusters, function(value, key){ - // var clusterQueryData = $scope.queryData[key].clone() || []; - // if (clusterQueryData.length == 50) { - // clusterQueryData.shift(); - // for(var i = 0; i < clusterQueryData.length; i++) { - // clusterQueryData[i] = [i, clusterQueryData[1]]; - // } - // } - // clusterQueryData.push(value); - // $scope.queryData[key] = clusterQueryData; - // chartData.push(clusterQueryData); - // }); - // $scope.queryData.chart = chartData; - // $timeout(query_tick, 10000); - // }).error(function(){ - // console.log("Unable to update queries"); - // $timeout(query_tick, 30000); - // }); - // })(); + (function query_tick() { + $http.get('/service/dashboard/query/status').success(function (data) { + $scope.queryData.errMsg = data.errMsg; + $scope.hasslowwarnings = data.slowQueries > 0; + + var now = new Date(); + now.setMilliseconds(0); + now.setSeconds(0); + var nowTime = now.getTime(); + + + var counterCopy = angular.copy($scope.queryData.counters); + angular.forEach(data.clusters, function(cluster) { + var clusterData = null; + // Find existing cluster's data + angular.forEach(counterCopy, function(series){ + if (series.label == cluster) { + clusterData = series.data; + } + }); + + // New cluster add and continue populating + if (clusterData == null) { + clusterData = []; + counterCopy.push({data:clusterData, label:cluster}); + } + + // Update counts for each time tick + angular.forEach(data.counts[cluster], function(count, time){ + var plot = null; + angular.forEach(clusterData, function(plotterPoint) { + if (plotterPoint[0] == time) { + plot = plotterPoint; + } + }); + + // New time, plot it + if (plot == null) { + clusterData.push([time, count]); + } else if (plot[1] < count) { + plot[1] = count; + } + }); + }); + angular.forEach(counterCopy, function(cluster){ + if (cluster['data'].length > 10) { + cluster['data'] = cluster['data'].slice(); + } + }); + $scope.queryData.counters = counterCopy; + + var perfCopy = angular.copy($scope.queryData.performance); + angular.forEach(data.clusters, function(cluster) { + var clusterData = null; + // Find existing cluster's data + angular.forEach(perfCopy, function(series){ + if (series.label == cluster) { + clusterData = series.data; + } + }); + + // New cluster add and continue populating + if (clusterData == null) { + clusterData = []; + perfCopy.push({data:clusterData, label:cluster}); + } + + // Update counts for each time tick + angular.forEach(data.performance[cluster], function(perf, time){ + var plot = null; + angular.forEach(clusterData, function(plotterPoint) { + if (plotterPoint[0] == time) { + plot = plotterPoint; + } + }); + + // New time, plot it + if (plot == null) { + clusterData.push([time, perf]); + } + }); + }); + angular.forEach(perfCopy, function(cluster){ + if (cluster['data'].length > 10) { + cluster['data'] = cluster['data'].slice(); + } + }); + $scope.queryData.performance = perfCopy; + $scope.queryData.clusters = data.clusters; + + $timeout(query_tick, 10000); + }).error(function(){ + console.log("Unable to update queries"); + $timeout(query_tick, 10000); + }); + })(); }) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/f102d62a/contrib/blur-console/src/main/resources/org/apache/blur/console/webapp/js/directives.js ---------------------------------------------------------------------- diff --git a/contrib/blur-console/src/main/resources/org/apache/blur/console/webapp/js/directives.js b/contrib/blur-console/src/main/resources/org/apache/blur/console/webapp/js/directives.js index 1a609ca..8b823f8 100644 --- a/contrib/blur-console/src/main/resources/org/apache/blur/console/webapp/js/directives.js +++ b/contrib/blur-console/src/main/resources/org/apache/blur/console/webapp/js/directives.js @@ -161,7 +161,8 @@ blurApp.directive('linechart', ['flotrService', '$window', function(flotrService restrict: 'EA', scope: { chartdata: '=', - title: '@' + title: '@', + xaxismode: '@' }, link: function(scope, element, attrs) { flotrService.flotr().then(function(Flotr) { @@ -191,7 +192,15 @@ blurApp.directive('linechart', ['flotrService', '$window', function(flotrService element.css('min-height', 200); element.css('min-weight', 200); element.css('max-height', 500); - element.css('max-width', 500); + element.css('max-width', 1000); + + var xaxisobj = {} + if (scope.xaxismode === 'time') { + xaxisobj['mode'] = 'time' + xaxisobj['timeMode'] = 'Local' + xaxisobj['timeFormat'] = "%h:%M" + xaxisobj['noTicks'] = data[0]['data'].length == 0 ? 1 : data[0]['data'].length + } Flotr.draw(element[0], data, { title: scope.title, @@ -199,9 +208,7 @@ blurApp.directive('linechart', ['flotrService', '$window', function(flotrService min: 0, tickDecimals: 0 }, - xaxis: { - - } + xaxis: xaxisobj }); } }); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/f102d62a/contrib/blur-console/src/main/resources/org/apache/blur/console/webapp/partials/dashboard.tpl.html ---------------------------------------------------------------------- diff --git a/contrib/blur-console/src/main/resources/org/apache/blur/console/webapp/partials/dashboard.tpl.html b/contrib/blur-console/src/main/resources/org/apache/blur/console/webapp/partials/dashboard.tpl.html index 70a1a84..829e84e 100644 --- a/contrib/blur-console/src/main/resources/org/apache/blur/console/webapp/partials/dashboard.tpl.html +++ b/contrib/blur-console/src/main/resources/org/apache/blur/console/webapp/partials/dashboard.tpl.html @@ -46,7 +46,7 @@ Licensed to the Apache Software Foundation (ASF) under one <div class="col-md-12"> <h4>Queries By Cluster</h4> <div ng-if="!queryData.errmsg"> - <div linechart chartdata="queryData.chart"></div> + <div linechart chartdata="queryData.counters" xaxismode="time"></div> </div> <div ng-if="queryData.errmsg"> <div class="alert alert-danger">{{queryData.errmsg}}</div> @@ -56,6 +56,12 @@ Licensed to the Apache Software Foundation (ASF) under one <div class="row"> <div class="col-md-12"> <h4>Query Performance By Cluster</h4> + <div ng-if="!queryData.errmsg"> + <div linechart chartdata="queryData.performance" xaxismode="time"></div> + </div> + <div ng-if="queryData.errmsg"> + <div class="alert alert-danger">{{queryData.errmsg}}</div> + </div> </div> </div> </div>
