Changes needed for live updates.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/b0eda0de Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/b0eda0de Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/b0eda0de Branch: refs/heads/master Commit: b0eda0de39cd0726d96749d53d287f3087aa2785 Parents: dcee9d6 Author: Aaron McCurry <[email protected]> Authored: Mon Sep 10 22:38:05 2012 -0400 Committer: Aaron McCurry <[email protected]> Committed: Mon Sep 10 22:38:05 2012 -0400 ---------------------------------------------------------------------- .../java/org/apache/blur/manager/IndexManager.java | 24 + .../apache/blur/thrift/ThriftBlurShardServer.java | 37 +- .../java/org/apache/blur/gui/HttpJettyServer.java | 2 +- .../org/apache/blur/gui/LiveMetricsServlet.java | 61 + src/blur-gui/src/main/webapp/d3.v2.js |16001 ++++++--------- src/blur-gui/src/main/webapp/home.jsp | 6 +- src/blur-gui/src/main/webapp/livemetrics.jsp | 111 + src/blur-gui/src/main/webapp/metrics.jsp | 6 +- src/blur-gui/src/main/webapp/shardList.jsp | 6 +- src/blur-gui/src/main/webapp/table.jsp | 6 +- .../java/org/apache/blur/metrics/HeapMetrics.java | 81 + .../java/org/apache/blur/metrics/QueryMetrics.java | 142 + .../org/apache/blur/metrics/SystemLoadMetrics.java | 72 + 13 files changed, 7339 insertions(+), 9216 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0eda0de/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java b/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java index 2bbb646..0470db5 100644 --- a/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java +++ b/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java @@ -57,6 +57,7 @@ import org.apache.blur.manager.status.QueryStatus; import org.apache.blur.manager.status.QueryStatusManager; import org.apache.blur.manager.writer.BlurIndex; import org.apache.blur.metrics.BlurMetrics; +import org.apache.blur.metrics.QueryMetrics; import org.apache.blur.thrift.BException; import org.apache.blur.thrift.MutationHelper; import org.apache.blur.thrift.generated.BlurException; @@ -120,7 +121,9 @@ public class IndexManager { private BlurPartitioner<BytesWritable, Void> _blurPartitioner = new BlurPartitioner<BytesWritable, Void>(); private BlurFilterCache _filterCache = new DefaultBlurFilterCache(); private BlurMetrics _blurMetrics; + private QueryMetrics _queryMetrics; private long _defaultParallelCallTimeout = TimeUnit.MINUTES.toMillis(1); + public void setMaxClauseCount(int maxClauseCount) { BooleanQuery.setMaxClauseCount(maxClauseCount); @@ -131,6 +134,7 @@ public class IndexManager { // @TODO give the mutate it's own thread pool _mutateExecutor = Executors.newThreadPool("index-manager-mutate", _threadCount); _statusManager.init(); + _queryMetrics = QueryMetrics.getInstance(); LOG.info("Init Complete"); } @@ -179,7 +183,9 @@ public class IndexManager { IndexReader reader = null; try { reader = index.getIndexReader(); + long s = System.nanoTime(); fetchRow(reader, table, selector, fetchResult); + _queryMetrics.recordDataFetch(System.nanoTime() - s, getRecordCount(fetchResult)); if (_blurMetrics != null) { if (fetchResult.rowResult != null) { if (fetchResult.rowResult.row != null && fetchResult.rowResult.row.records != null) { @@ -205,6 +211,17 @@ public class IndexManager { } } + private long getRecordCount(FetchResult fetchResult) { + if (fetchResult.rowResult != null) { + if (fetchResult.rowResult.row != null && fetchResult.rowResult.row.records != null) { + return fetchResult.rowResult.row.records.size(); + } + } else if (fetchResult.recordResult != null) { + return 1; + } + return 0; + } + private void populateSelector(String table, Selector selector) throws IOException, BlurException { String rowId = selector.rowId; String recordId = selector.recordId; @@ -286,6 +303,7 @@ public class IndexManager { } public BlurResultIterable query(final String table, final BlurQuery blurQuery, AtomicLongArray facetedCounts) throws Exception { + long s = System.nanoTime(); final AtomicBoolean running = new AtomicBoolean(true); final QueryStatus status = _statusManager.newQueryStatus(table, blurQuery, _threadCount, running); _blurMetrics.queriesExternal.incrementAndGet(); @@ -327,6 +345,7 @@ public class IndexManager { }).merge(merger); } finally { _statusManager.removeStatus(status); + _queryMetrics.recordQuery(System.nanoTime() - s); } } @@ -730,16 +749,19 @@ public class IndexManager { waitVisiblity = waitToBeVisible; } RowMutationType type = mutation.rowMutationType; + long start = System.nanoTime(); switch (type) { case REPLACE_ROW: Row row = MutationHelper.getRowFromMutations(mutation.rowId, mutation.recordMutations); blurIndex.replaceRow(waitVisiblity, mutation.wal, row); + _queryMetrics.recordDataMutate(System.nanoTime() - start, row.records.size()); break; case UPDATE_ROW: doUpdateRowMutation(mutation, blurIndex); break; case DELETE_ROW: blurIndex.deleteRow(waitVisiblity, mutation.wal, mutation.rowId); + _queryMetrics.recordDataMutate(System.nanoTime() - start, 1); break; default: throw new RuntimeException("Not supported [" + type + "]"); @@ -842,8 +864,10 @@ public class IndexManager { } } + long s = System.nanoTime(); // Finally, replace the existing row with the new row we have built. blurIndex.replaceRow(mutation.waitToBeVisible, mutation.wal, newRow); + _queryMetrics.recordDataMutate(System.nanoTime() - s, newRow.records.size()); } else { throw new BException("Mutation cannot update row that does not exist.", mutation); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0eda0de/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java index d92c394..7a6ea44 100644 --- a/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java +++ b/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java @@ -101,7 +101,28 @@ public class ThriftBlurShardServer extends ThriftServer { int slabCount = configuration.getInt(BLUR_SHARD_BLOCKCACHE_SLAB_COUNT, 1); Cache cache; Configuration config = new Configuration(); + + String bindAddress = configuration.get(BLUR_SHARD_BIND_ADDRESS); + int bindPort = configuration.getInt(BLUR_SHARD_BIND_PORT, -1); + bindPort += serverIndex; + BlurMetrics blurMetrics = new BlurMetrics(config); + + int baseGuiPort = Integer.parseInt(configuration.get(BLUR_GUI_SHARD_PORT)); + final HttpJettyServer httpServer; + if (baseGuiPort > 0) { + int webServerPort = baseGuiPort + serverIndex; + + // TODO: this got ugly, there has to be a better way to handle all these + // params + // without reversing the mvn dependancy and making blur-gui on top. + httpServer = new HttpJettyServer(bindPort, webServerPort, configuration.getInt(BLUR_CONTROLLER_BIND_PORT, -1), configuration.getInt(BLUR_SHARD_BIND_PORT, -1), + configuration.getInt(BLUR_GUI_CONTROLLER_PORT, -1), configuration.getInt(BLUR_GUI_SHARD_PORT, -1), "shard", blurMetrics); + } else { + httpServer = null; + } + + if (slabCount >= 1) { BlockCache blockCache; boolean directAllocation = configuration.getBoolean(BLUR_SHARD_BLOCKCACHE_DIRECT_MEMORY_ALLOCATION, true); @@ -128,9 +149,6 @@ public class ThriftBlurShardServer extends ThriftServer { cache = BlockDirectory.NO_CACHE; } - String bindAddress = configuration.get(BLUR_SHARD_BIND_ADDRESS); - int bindPort = configuration.getInt(BLUR_SHARD_BIND_PORT, -1); - bindPort += serverIndex; LOG.info("Shard Server using index [{0}] bind address [{1}]", serverIndex, bindAddress + ":" + bindPort); @@ -209,19 +227,6 @@ public class ThriftBlurShardServer extends ThriftServer { server.setIface(iface); server.setConfiguration(configuration); - int baseGuiPort = Integer.parseInt(configuration.get(BLUR_GUI_SHARD_PORT)); - final HttpJettyServer httpServer; - if (baseGuiPort > 0) { - int webServerPort = baseGuiPort + serverIndex; - - // TODO: this got ugly, there has to be a better way to handle all these - // params - // without reversing the mvn dependancy and making blur-gui on top. - httpServer = new HttpJettyServer(bindPort, webServerPort, configuration.getInt(BLUR_CONTROLLER_BIND_PORT, -1), configuration.getInt(BLUR_SHARD_BIND_PORT, -1), - configuration.getInt(BLUR_GUI_CONTROLLER_PORT, -1), configuration.getInt(BLUR_GUI_SHARD_PORT, -1), "shard", blurMetrics); - } else { - httpServer = null; - } // This will shutdown the server when the correct path is set in zk BlurShutdown shutdown = new BlurShutdown() { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0eda0de/src/blur-gui/src/main/java/org/apache/blur/gui/HttpJettyServer.java ---------------------------------------------------------------------- diff --git a/src/blur-gui/src/main/java/org/apache/blur/gui/HttpJettyServer.java b/src/blur-gui/src/main/java/org/apache/blur/gui/HttpJettyServer.java index 37b51ef..d775bb8 100644 --- a/src/blur-gui/src/main/java/org/apache/blur/gui/HttpJettyServer.java +++ b/src/blur-gui/src/main/java/org/apache/blur/gui/HttpJettyServer.java @@ -27,7 +27,6 @@ import org.mortbay.jetty.Server; import org.mortbay.jetty.servlet.ServletHolder; import org.mortbay.jetty.webapp.WebAppContext; - /** * Starts up a Jetty server to run the utility gui. * @@ -76,6 +75,7 @@ public class HttpJettyServer { context.setWar(warPath); context.setContextPath("/"); context.setParentLoaderPriority(true); + context.addServlet(new ServletHolder(new LiveMetricsServlet()), "/livemetrics"); context.addServlet(new ServletHolder(new MetricsServlet(bm)), "/metrics"); context.addServlet(new ServletHolder(new LogServlet(blurLogFile)), "/logs"); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0eda0de/src/blur-gui/src/main/java/org/apache/blur/gui/LiveMetricsServlet.java ---------------------------------------------------------------------- diff --git a/src/blur-gui/src/main/java/org/apache/blur/gui/LiveMetricsServlet.java b/src/blur-gui/src/main/java/org/apache/blur/gui/LiveMetricsServlet.java new file mode 100644 index 0000000..0a234f6 --- /dev/null +++ b/src/blur-gui/src/main/java/org/apache/blur/gui/LiveMetricsServlet.java @@ -0,0 +1,61 @@ +package org.apache.blur.gui; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import java.io.PrintWriter; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.blur.metrics.HeapMetrics; +import org.apache.blur.metrics.QueryMetrics; +import org.apache.blur.metrics.SystemLoadMetrics; + +public class LiveMetricsServlet extends HttpServlet { + + private static final long serialVersionUID = 1L; + + private HeapMetrics heapMetrics; + private SystemLoadMetrics systemLoadMetrics; + private QueryMetrics queryMetrics; + + public LiveMetricsServlet() { + heapMetrics = HeapMetrics.getInstance(); + systemLoadMetrics = SystemLoadMetrics.getInstance(); + queryMetrics = QueryMetrics.getInstance(); + } + + protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { + response.setContentType("application/json"); + PrintWriter out = response.getWriter(); + out.write("{"); + out.write("\"jvm\":{\"xLabel\":\"Time\",\"yLabel\":\"Heap (GB)\",\"lines\":"); + heapMetrics.writeJson(out); + out.write("}"); + out.write(",\"blur\":{\"xLabel\":\"Time\",\"yLabel\":\"Rates\",\"lines\":"); + queryMetrics.writeJson(out); + out.write("}"); + out.write(",\"system\":{\"xLabel\":\"Time\",\"yLabel\":\"Load\",\"lines\":"); + systemLoadMetrics.writeJson(out); + out.write("}"); + out.write("}"); + } + +}
