Started building out the Query charts
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/5663ec3e Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/5663ec3e Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/5663ec3e Branch: refs/heads/blur-console-v2 Commit: 5663ec3e5d237d2bc891267f6edcb4ba0d5eb6c4 Parents: 710e6c1 Author: Chris Rohr <[email protected]> Authored: Wed Oct 23 20:48:35 2013 -0400 Committer: Chris Rohr <[email protected]> Committed: Wed Oct 23 20:48:35 2013 -0400 ---------------------------------------------------------------------- .../blur/console/servlets/DashboardServlet.java | 19 ++-- .../org/apache/blur/console/util/QueryUtil.java | 53 ++++++++++ .../apache/blur/console/webapp/js/dashboard.js | 26 +++++ .../apache/blur/console/webapp/js/directives.js | 53 ++++++++++ .../console/webapp/partials/dashboard.tpl.html | 6 ++ .../apache/blur/console/ConsoleTestBase.java | 9 +- .../apache/blur/console/util/QueryUtilTest.java | 101 +++++++++++++++++++ .../apache/blur/console/util/TableUtilTest.java | 4 +- 8 files changed, 258 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5663ec3e/contrib/blur-console/src/main/java/org/apache/blur/console/servlets/DashboardServlet.java ---------------------------------------------------------------------- diff --git a/contrib/blur-console/src/main/java/org/apache/blur/console/servlets/DashboardServlet.java b/contrib/blur-console/src/main/java/org/apache/blur/console/servlets/DashboardServlet.java index c813c0f..562bfce 100644 --- a/contrib/blur-console/src/main/java/org/apache/blur/console/servlets/DashboardServlet.java +++ b/contrib/blur-console/src/main/java/org/apache/blur/console/servlets/DashboardServlet.java @@ -28,11 +28,10 @@ import javax.servlet.http.HttpServletResponse; import org.apache.blur.console.util.HttpUtil; import org.apache.blur.console.util.NodeUtil; +import org.apache.blur.console.util.QueryUtil; import org.apache.blur.console.util.TableUtil; import org.apache.commons.io.IOUtils; import org.codehaus.jackson.map.ObjectMapper; -import org.json.JSONException; -import org.json.JSONObject; public class DashboardServlet extends HttpServlet { private static final long serialVersionUID = 6522056391102413432L; @@ -44,7 +43,7 @@ public class DashboardServlet extends HttpServlet { sendNodeStatus(response); } else if ("/table/status".equalsIgnoreCase(path)) { sendTableStatus(response); - } else if ("query/status".equalsIgnoreCase(path)) { + } else if ("/query/status".equalsIgnoreCase(path)) { sendQueryStatus(response); } else { response.setStatus(404); @@ -93,15 +92,17 @@ public class DashboardServlet extends HttpServlet { } private void sendQueryStatus(HttpServletResponse response) throws IOException { - JSONObject nodeData = new JSONObject(); - + Map<String, Object> queryData = new HashMap<String, Object>(); + try { - nodeData.put("fill", "me"); - } catch (JSONException e) { + queryData = QueryUtil.getQueryStatus(); + } catch (IOException e) { + throw new IOException(e); + } catch (Exception e) { sendError(response, e); return; } - - HttpUtil.sendResponse(response, nodeData.toString(), HttpUtil.JSON); + + HttpUtil.sendResponse(response, new ObjectMapper().writeValueAsString(queryData), HttpUtil.JSON); } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5663ec3e/contrib/blur-console/src/main/java/org/apache/blur/console/util/QueryUtil.java ---------------------------------------------------------------------- diff --git a/contrib/blur-console/src/main/java/org/apache/blur/console/util/QueryUtil.java b/contrib/blur-console/src/main/java/org/apache/blur/console/util/QueryUtil.java new file mode 100644 index 0000000..15f295f --- /dev/null +++ b/contrib/blur-console/src/main/java/org/apache/blur/console/util/QueryUtil.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.blur.console.util; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.blur.thirdparty.thrift_0_9_0.TException; +import org.apache.blur.thrift.BlurClient; +import org.apache.blur.thrift.generated.Blur.Iface; +import org.apache.blur.thrift.generated.BlurException; + +public class QueryUtil { + + public static Map<String, Object> getQueryStatus() throws IOException, BlurException, TException { + Iface client = BlurClient.getClient(Config.getConnectionString()); + + Map<String, Integer> clusterQueryCount = new HashMap<String, Integer>(); + List<String> clusters = client.shardClusterList(); + + for (String cluster : clusters) { + List<String> tables = client.tableListByCluster(cluster); + int queryCount = 0; + + for (String table : tables) { + List<String> queryIds = client.queryStatusIdList(table); + queryCount += queryIds.size(); + } + clusterQueryCount.put(cluster, queryCount); + } + + System.out.println(clusterQueryCount); + + return null; + } +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5663ec3e/contrib/blur-console/src/main/resources/org/apache/blur/console/webapp/js/dashboard.js ---------------------------------------------------------------------- diff --git a/contrib/blur-console/src/main/resources/org/apache/blur/console/webapp/js/dashboard.js b/contrib/blur-console/src/main/resources/org/apache/blur/console/webapp/js/dashboard.js index f7e361d..8f4530a 100644 --- a/contrib/blur-console/src/main/resources/org/apache/blur/console/webapp/js/dashboard.js +++ b/contrib/blur-console/src/main/resources/org/apache/blur/console/webapp/js/dashboard.js @@ -22,6 +22,7 @@ blurApp.controller('DashboardCtrl', function($scope, $http, $timeout) { $scope.nodeData = {}; $scope.tableData = {}; + $scope.queryData = {}; (function node_tick() { $http.get('/service/dashboard/node/status').success(function (data) { @@ -42,4 +43,29 @@ blurApp.controller('DashboardCtrl', function($scope, $http, $timeout) { $timeout(table_tick, 10000); }); })(); + + (function query_tick() { + $http.get('/service/dashboard/query/status').success(function (data) { + $scope.queryData.errMsg = data.errMsg; + + var chartData = []; + angular.forEach(data.clusters, function(value, key){ + var clusterQueryData = $scope.queryData[key].clone() || []; + if (clusterQueryData.length == 50) { + clusterQueryData.shift(); + for(var i = 0; i < clusterQueryData.length; i++) { + clusterQueryData[i] = [i, clusterQueryData[1]]; + } + } + clusterQueryData.push(value); + $scope.queryData[key] = clusterQueryData; + chartData.push(clusterQueryData); + }); + $scope.queryData.chart = chartData; + $timeout(query_tick, 10000); + }).error(function(){ + console.log("Unable to update queries"); + $timeout(query_tick, 30000); + }); + })(); }) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5663ec3e/contrib/blur-console/src/main/resources/org/apache/blur/console/webapp/js/directives.js ---------------------------------------------------------------------- diff --git a/contrib/blur-console/src/main/resources/org/apache/blur/console/webapp/js/directives.js b/contrib/blur-console/src/main/resources/org/apache/blur/console/webapp/js/directives.js index 5f06802..4eeb308 100644 --- a/contrib/blur-console/src/main/resources/org/apache/blur/console/webapp/js/directives.js +++ b/contrib/blur-console/src/main/resources/org/apache/blur/console/webapp/js/directives.js @@ -150,4 +150,57 @@ blurApp.directive('stackedbarchart', ['flotrService', '$window', function(flotrS }); } }; +}]); + +blurApp.directive('linechart', ['flotrService', '$window', function(flotrService, $window){ + return { + restrict: 'EA', + scope: { + chartdata: '=', + title: '@' + }, + link: function(scope, element, attrs) { + flotrService.flotr().then(function(Flotr) { + window.onresize = function() { + scope.$apply(); + }; + + scope.$watch(function() { + return angular.element($window)[0].innerWidth; + }, function() { + scope.render(scope.chartdata); + }); + + scope.$watch('chartdata', function(newVals, oldVals) { + return scope.render(newVals); + }, true); + + scope.render = function(data) { + element.children().remove(); + if (!data) return; + + var width = element.parent()[0].clientWidth - 150, + height = width; + + element.css('height', height); + element.css('width', width); + element.css('min-height', 200); + element.css('min-weight', 200); + element.css('max-height', 500); + element.css('max-width', 500); + + Flotr.draw(element[0], data, { + title: scope.title, + yaxis: { + min: 0, + tickDecimals: 0 + }, + xaxis: { + + } + }); + } + }); + } + }; }]); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5663ec3e/contrib/blur-console/src/main/resources/org/apache/blur/console/webapp/partials/dashboard.tpl.html ---------------------------------------------------------------------- diff --git a/contrib/blur-console/src/main/resources/org/apache/blur/console/webapp/partials/dashboard.tpl.html b/contrib/blur-console/src/main/resources/org/apache/blur/console/webapp/partials/dashboard.tpl.html index b5060b8..2e6a12f 100644 --- a/contrib/blur-console/src/main/resources/org/apache/blur/console/webapp/partials/dashboard.tpl.html +++ b/contrib/blur-console/src/main/resources/org/apache/blur/console/webapp/partials/dashboard.tpl.html @@ -45,6 +45,12 @@ Licensed to the Apache Software Foundation (ASF) under one <div class="row"> <div class="col-md-12"> <h4>Queries By Cluster</h4> + <div ng-if="!queryData.errmsg"> + <div linechart chartdata="queryData.chart"></div> + </div> + <div ng-if="queryData.errmsg"> + <div class="alert alert-danger">{{queryData.errmsg}}</div> + </div> </div> </div> <div class="row"> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5663ec3e/contrib/blur-console/src/test/java/org/apache/blur/console/ConsoleTestBase.java ---------------------------------------------------------------------- diff --git a/contrib/blur-console/src/test/java/org/apache/blur/console/ConsoleTestBase.java b/contrib/blur-console/src/test/java/org/apache/blur/console/ConsoleTestBase.java index f012769..4d34fc2 100644 --- a/contrib/blur-console/src/test/java/org/apache/blur/console/ConsoleTestBase.java +++ b/contrib/blur-console/src/test/java/org/apache/blur/console/ConsoleTestBase.java @@ -22,11 +22,13 @@ import java.io.IOException; import org.apache.blur.MiniCluster; import org.apache.blur.console.util.Config; +import org.apache.commons.io.FileUtils; import org.junit.AfterClass; import org.junit.BeforeClass; public class ConsoleTestBase { protected static MiniCluster cluster; + protected static String TABLE_PATH = new File("./test-data/test-tables").getAbsolutePath(); @BeforeClass public static void startup() { @@ -35,9 +37,12 @@ public class ConsoleTestBase { } @AfterClass - public static void shutdown() { + public static void shutdown() throws IOException { cluster.shutdownBlurCluster(); - new File("./test-data").delete(); + File file = new File("./test-data"); + if (file.exists()) { + FileUtils.deleteDirectory(file); + } } protected void setupConfigIfNeeded() throws IOException { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5663ec3e/contrib/blur-console/src/test/java/org/apache/blur/console/util/QueryUtilTest.java ---------------------------------------------------------------------- diff --git a/contrib/blur-console/src/test/java/org/apache/blur/console/util/QueryUtilTest.java b/contrib/blur-console/src/test/java/org/apache/blur/console/util/QueryUtilTest.java new file mode 100644 index 0000000..923aec8 --- /dev/null +++ b/contrib/blur-console/src/test/java/org/apache/blur/console/util/QueryUtilTest.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.blur.console.util; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + +import org.apache.blur.console.ConsoleTestBase; +import org.apache.blur.thirdparty.thrift_0_9_0.TException; +import org.apache.blur.thrift.BlurClient; +import org.apache.blur.thrift.generated.Blur.Iface; +import org.apache.blur.thrift.generated.BlurException; +import org.apache.blur.thrift.generated.BlurQuery; +import org.apache.blur.thrift.generated.Column; +import org.apache.blur.thrift.generated.Query; +import org.apache.blur.thrift.generated.Record; +import org.apache.blur.thrift.generated.RecordMutation; +import org.apache.blur.thrift.generated.RecordMutationType; +import org.apache.blur.thrift.generated.RowMutation; +import org.apache.blur.thrift.generated.RowMutationType; +import org.apache.blur.thrift.generated.ScoreType; +import org.apache.blur.thrift.generated.TableDescriptor; +import org.junit.Before; +import org.junit.Test; + +public class QueryUtilTest extends ConsoleTestBase { + @Before + public void setup() throws IOException, BlurException, TException { + setupConfigIfNeeded(); + + Iface client = BlurClient.getClient(cluster.getControllerConnectionStr()); + + TableDescriptor td = new TableDescriptor(); + td.setShardCount(11); + td.setTableUri("file://" + TABLE_PATH + "/queryUnitTable"); + td.setCluster("default"); + td.setName("queryUnitTable"); + td.setEnabled(true); + client.createTable(td); + +// Record record = new Record(); +// record.setRecordId("abcd"); +// record.setFamily("fam0"); +// List<Column> columns = new ArrayList<Column>(); +// columns.add(new Column("col0", "testvalue")); +// record.setColumns(columns); +// +// RecordMutation recordMutation = new RecordMutation(); +// recordMutation.setRecord(record); +// recordMutation.setRecordMutationType(RecordMutationType.REPLACE_ENTIRE_RECORD); +// +// List<RecordMutation> recordMutations = new ArrayList<RecordMutation>(); +// recordMutations.add(recordMutation); +// +// RowMutation mutation = new RowMutation(); +// mutation.setTable("unitTable); +// mutation.setRowId(rowid); +// mutation.setRowMutationType(RowMutationType.UPDATE_ROW); +// mutation.setRecordMutations(recordMutations); +// +// client.mutate(mutation); + + Record record = new Record("abcd", "fam0", Arrays.asList(new Column[]{ new Column("col0", "testvalue")})); + RecordMutation recordMutation = new RecordMutation(RecordMutationType.REPLACE_ENTIRE_RECORD, record); + RowMutation rowMutation = new RowMutation("queryUnitTable", "12345", false, RowMutationType.REPLACE_ROW, Arrays.asList(new RecordMutation[]{ recordMutation }), true); + client.mutate(rowMutation); + } + + @Test + public void testGetQueryStatus() throws BlurException, IOException, TException { + Iface client = BlurClient.getClient(cluster.getControllerConnectionStr()); + QueryUtil.getQueryStatus(); + + BlurQuery query = new BlurQuery( + new Query("fam0.col0:*", true, ScoreType.SUPER, null, null), + null, + null, //new Selector(false, null, null, null, null, null, 0, 10, null), + false, 0, 10, 1, 2000, UUID.randomUUID().toString(), "testUser", false, System.currentTimeMillis()); + client.query("queryUnitTable", query); + + QueryUtil.getQueryStatus(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5663ec3e/contrib/blur-console/src/test/java/org/apache/blur/console/util/TableUtilTest.java ---------------------------------------------------------------------- diff --git a/contrib/blur-console/src/test/java/org/apache/blur/console/util/TableUtilTest.java b/contrib/blur-console/src/test/java/org/apache/blur/console/util/TableUtilTest.java index ce5bc94..7a84b0d 100644 --- a/contrib/blur-console/src/test/java/org/apache/blur/console/util/TableUtilTest.java +++ b/contrib/blur-console/src/test/java/org/apache/blur/console/util/TableUtilTest.java @@ -65,9 +65,9 @@ public class TableUtilTest extends ConsoleTestBase { TableDescriptor td = new TableDescriptor(); td.setShardCount(11); - td.setTableUri("file:///tmp/unitTable"); + td.setTableUri("file://" + TABLE_PATH + "/tableUnitTable"); td.setCluster("default"); - td.setName("unitTable"); + td.setName("tableUnitTable"); td.setEnabled(true); client.createTable(td);
