Removed the last of the old metrics.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/cdbe70f6 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/cdbe70f6 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/cdbe70f6 Branch: refs/heads/0.2-dev Commit: cdbe70f6adaf9a0d1e68916706038855e6fa3060 Parents: e3a319d Author: Aaron McCurry <[email protected]> Authored: Wed Mar 6 15:17:30 2013 -0500 Committer: Aaron McCurry <[email protected]> Committed: Wed Mar 6 15:17:30 2013 -0500 ---------------------------------------------------------------------- .../indexserver/DistributedIndexServer.java | 10 +- .../org/apache/blur/thrift/ThriftBlurServer.java | 10 +- src/blur-gui/pom.xml | 35 ++-- .../java/org/apache/blur/gui/HttpJettyServer.java | 12 +- .../org/apache/blur/gui/LiveMetricsServlet.java | 10 - .../java/org/apache/blur/gui/MetricsServlet.java | 51 ---- .../org/apache/blur/store/hdfs/HdfsDirectory.java | 17 +- .../java/org/apache/blur/metrics/BlurMetrics.java | 129 ---------- .../java/org/apache/blur/metrics/HeapMetrics.java | 107 -------- .../org/apache/blur/metrics/MetricsConstants.java | 1 + .../java/org/apache/blur/metrics/QueryMetrics.java | 199 --------------- .../org/apache/blur/metrics/SystemLoadMetrics.java | 89 ------- 12 files changed, 35 insertions(+), 635 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/cdbe70f6/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java b/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java index 1429b9b..a3fa7cb 100644 --- a/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java +++ b/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java @@ -56,7 +56,6 @@ import org.apache.blur.manager.writer.BlurIndexRefresher; import org.apache.blur.manager.writer.BlurNRTIndex; import org.apache.blur.manager.writer.SharedMergeScheduler; import org.apache.blur.metrics.AtomicLongGauge; -import org.apache.blur.metrics.BlurMetrics; import org.apache.blur.server.ShardContext; import org.apache.blur.server.TableContext; import org.apache.blur.server.ZooKeeperTypeManager; @@ -102,7 +101,6 @@ public class DistributedIndexServer extends AbstractIndexServer { private String _nodeName; private int _shardOpenerThreadCount; private BlurIndexRefresher _refresher; - private BlurMetrics _blurMetrics; private ZooKeeper _zookeeper; private String _cluster; @@ -253,7 +251,7 @@ public class DistributedIndexServer extends AbstractIndexServer { LOG.debug("Indexes fetched for table [{0}].", table); int count = indexes.size(); indexCount += count; - updateMetrics(_blurMetrics, indexes, segmentCount, indexMemoryUsage); + updateMetrics(indexes, segmentCount, indexMemoryUsage); LOG.debug("Table [{0}] has [{1}] number of shards online in this node.", table, count); } catch (IOException e) { LOG.error("Unknown error trying to warm table [{0}]", e, table); @@ -265,7 +263,7 @@ public class DistributedIndexServer extends AbstractIndexServer { } } - private void updateMetrics(BlurMetrics blurMetrics, Map<String, BlurIndex> indexes, AtomicLong segmentCount, AtomicLong indexMemoryUsage) throws IOException { + private void updateMetrics(Map<String, BlurIndex> indexes, AtomicLong segmentCount, AtomicLong indexMemoryUsage) throws IOException { // @TODO not sure how to do this yet // for (BlurIndex index : indexes.values()) { // IndexReader reader = index.getIndexReader(); @@ -696,10 +694,6 @@ public class DistributedIndexServer extends AbstractIndexServer { _refresher = refresher; } - public void setBlurMetrics(BlurMetrics blurMetrics) { - _blurMetrics = blurMetrics; - } - public void setZookeeper(ZooKeeper zookeeper) { _zookeeper = zookeeper; } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/cdbe70f6/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java b/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java index 7d8719a..1880436 100644 --- a/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java +++ b/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java @@ -18,7 +18,6 @@ package org.apache.blur.thrift; */ import static org.apache.blur.utils.BlurConstants.BLUR_CLUSTER; import static org.apache.blur.utils.BlurConstants.BLUR_CLUSTER_NAME; -import static org.apache.blur.utils.BlurConstants.BLUR_CONTROLLER_BIND_PORT; import static org.apache.blur.utils.BlurConstants.BLUR_GUI_CONTROLLER_PORT; import static org.apache.blur.utils.BlurConstants.BLUR_GUI_SHARD_PORT; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BIND_ADDRESS; @@ -127,18 +126,15 @@ public class ThriftBlurServer extends AbstractThriftServer { int bindPort = configuration.getInt(BLUR_SHARD_BIND_PORT, -1); bindPort += serverIndex; - // BlurMetrics blurMetrics = new BlurMetrics(config); - int baseGuiPort = Integer.parseInt(configuration.get(BLUR_GUI_SHARD_PORT)); final HttpJettyServer httpServer; if (baseGuiPort > 0) { int webServerPort = baseGuiPort + serverIndex; // TODO: this got ugly, there has to be a better way to handle all these - // params - // without reversing the mvn dependancy and making blur-gui on top. - httpServer = new HttpJettyServer(bindPort, webServerPort, configuration.getInt(BLUR_CONTROLLER_BIND_PORT, -1), configuration.getInt(BLUR_SHARD_BIND_PORT, -1), - configuration.getInt(BLUR_GUI_CONTROLLER_PORT, -1), configuration.getInt(BLUR_GUI_SHARD_PORT, -1), "blur", null); + // params without reversing the mvn dependancy and making blur-gui on top. + httpServer = new HttpJettyServer(bindPort, webServerPort, configuration.getInt(BLUR_SHARD_BIND_PORT, -1), configuration.getInt(BLUR_GUI_CONTROLLER_PORT, -1), + configuration.getInt(BLUR_GUI_SHARD_PORT, -1), "blur", null); } else { httpServer = null; } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/cdbe70f6/src/blur-gui/pom.xml ---------------------------------------------------------------------- diff --git a/src/blur-gui/pom.xml b/src/blur-gui/pom.xml index 1c24dc0..202fe98 100644 --- a/src/blur-gui/pom.xml +++ b/src/blur-gui/pom.xml @@ -1,22 +1,14 @@ <?xml version="1.0" encoding="UTF-8" ?> -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---> +<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor + license agreements. See the NOTICE file distributed with this work for additional + information regarding copyright ownership. The ASF licenses this file to + you under the Apache License, Version 2.0 (the "License"); you may not use + this file except in compliance with the License. You may obtain a copy of + the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required + by applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS + OF ANY KIND, either express or implied. See the License for the specific + language governing permissions and limitations under the License. --> <project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> <modelVersion>4.0.0</modelVersion> @@ -24,7 +16,7 @@ under the License. <groupId>org.apache.blur</groupId> <artifactId>blur</artifactId> <version>0.2.0-SNAPSHOT</version> - <relativePath>../pom.xml</relativePath> + <relativePath>../pom.xml</relativePath> </parent> <groupId>org.apache.blur</groupId> <artifactId>blur-gui</artifactId> @@ -52,6 +44,11 @@ under the License. <version>${project.version}</version> </dependency> <dependency> + <groupId>com.yammer.metrics</groupId> + <artifactId>metrics-servlet</artifactId> + <version>${metrics-core.version}</version> + </dependency> + <dependency> <groupId>log4j</groupId> <artifactId>log4j</artifactId> <version>${log4j.version}</version> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/cdbe70f6/src/blur-gui/src/main/java/org/apache/blur/gui/HttpJettyServer.java ---------------------------------------------------------------------- diff --git a/src/blur-gui/src/main/java/org/apache/blur/gui/HttpJettyServer.java b/src/blur-gui/src/main/java/org/apache/blur/gui/HttpJettyServer.java index 33ca980..e29a13f 100644 --- a/src/blur-gui/src/main/java/org/apache/blur/gui/HttpJettyServer.java +++ b/src/blur-gui/src/main/java/org/apache/blur/gui/HttpJettyServer.java @@ -22,7 +22,6 @@ import java.util.Properties; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; -import org.apache.blur.metrics.BlurMetrics; import org.apache.thrift.TProcessor; import org.apache.thrift.protocol.TJSONProtocol; import org.apache.thrift.protocol.TProtocolFactory; @@ -31,6 +30,8 @@ import org.mortbay.jetty.Server; import org.mortbay.jetty.servlet.ServletHolder; import org.mortbay.jetty.webapp.WebAppContext; +import com.yammer.metrics.reporting.MetricsServlet; + /** * Starts up a Jetty server to run the utility gui. * @@ -60,7 +61,7 @@ public class HttpJettyServer { * @param processor * @throws IOException */ - public HttpJettyServer(int bindPort, int port, int baseControllerPort, int baseShardPort, int baseGuiControllerPort, int baseGuiShardPort, String base, TProcessor processor) + public HttpJettyServer(int bindPort, int port, int baseRpcPort, int baseGuiControllerPort, int baseGuiShardPort, String base, TProcessor processor) throws IOException { server = new Server(port); @@ -68,8 +69,7 @@ public class HttpJettyServer { String logFile = System.getProperty("blur.log.file"); String blurLogFile = logDir + "/" + logFile; System.setProperty("blur.gui.servicing.port", bindPort + ""); - System.setProperty("blur.base.shard.port", baseShardPort + ""); - System.setProperty("blur.base.controller.port", baseControllerPort + ""); + System.setProperty("blur.base.shard.port", baseRpcPort + ""); System.setProperty("baseGuiShardPort", baseGuiShardPort + ""); System.setProperty("baseGuiControllerPort", baseGuiControllerPort + ""); System.setProperty("blur.gui.mode", base); @@ -82,10 +82,10 @@ public class HttpJettyServer { context.setParentLoaderPriority(true); TProtocolFactory protocolFactory = new TJSONProtocol.Factory(); if (processor != null) { - context.addServlet(new ServletHolder(new TServlet(processor, protocolFactory)), "/json"); + context.addServlet(new ServletHolder(new TServlet(processor, protocolFactory)), "/rpc"); } context.addServlet(new ServletHolder(new LiveMetricsServlet()), "/livemetrics"); -// context.addServlet(new ServletHolder(new MetricsServlet(bm)), "/metrics"); + context.addServlet(new ServletHolder(new MetricsServlet()), "/metrics"); context.addServlet(new ServletHolder(new LogServlet(blurLogFile)), "/logs"); LOG.info("WEB GUI coming up for resource: " + base); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/cdbe70f6/src/blur-gui/src/main/java/org/apache/blur/gui/LiveMetricsServlet.java ---------------------------------------------------------------------- diff --git a/src/blur-gui/src/main/java/org/apache/blur/gui/LiveMetricsServlet.java b/src/blur-gui/src/main/java/org/apache/blur/gui/LiveMetricsServlet.java index 956541d..0734f85 100644 --- a/src/blur-gui/src/main/java/org/apache/blur/gui/LiveMetricsServlet.java +++ b/src/blur-gui/src/main/java/org/apache/blur/gui/LiveMetricsServlet.java @@ -16,17 +16,7 @@ package org.apache.blur.gui; * See the License for the specific language governing permissions and * limitations under the License. */ -import java.io.IOException; -import java.io.PrintWriter; - -import javax.servlet.ServletException; import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.apache.blur.metrics.HeapMetrics; -import org.apache.blur.metrics.QueryMetrics; -import org.apache.blur.metrics.SystemLoadMetrics; public class LiveMetricsServlet extends HttpServlet { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/cdbe70f6/src/blur-gui/src/main/java/org/apache/blur/gui/MetricsServlet.java ---------------------------------------------------------------------- diff --git a/src/blur-gui/src/main/java/org/apache/blur/gui/MetricsServlet.java b/src/blur-gui/src/main/java/org/apache/blur/gui/MetricsServlet.java deleted file mode 100644 index c1bfd20..0000000 --- a/src/blur-gui/src/main/java/org/apache/blur/gui/MetricsServlet.java +++ /dev/null @@ -1,51 +0,0 @@ -package org.apache.blur.gui; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import java.io.IOException; -import java.io.PrintWriter; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.apache.blur.metrics.BlurMetrics; -import org.codehaus.jackson.map.ObjectMapper; - - -public class MetricsServlet extends HttpServlet { - - private static final long serialVersionUID = 1L; - - private BlurMetrics bm = null; - - public MetricsServlet() { - } - - public MetricsServlet(BlurMetrics bm) { - this.bm = bm; - } - - protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { - response.setContentType("application/json"); - PrintWriter out = response.getWriter(); - ObjectMapper mapper = new ObjectMapper(); - mapper.writeValue(out, bm); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/cdbe70f6/src/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java ---------------------------------------------------------------------- diff --git a/src/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java b/src/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java index ab6ad1b..0e92867 100644 --- a/src/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java +++ b/src/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java @@ -17,7 +17,7 @@ package org.apache.blur.store.hdfs; * limitations under the License. */ -import static org.apache.blur.metrics.MetricsConstants.BLUR; +import static org.apache.blur.metrics.MetricsConstants.HDFS; import static org.apache.blur.metrics.MetricsConstants.ORG_APACHE_BLUR; import java.io.FileNotFoundException; @@ -53,6 +53,8 @@ import com.yammer.metrics.core.Meter; import com.yammer.metrics.core.MetricName; public class HdfsDirectory extends Directory { + + private static final Log LOG = LogFactory.getLog(HdfsDirectory.class); private static AtomicLong deleteCounter = new AtomicLong(); @@ -106,11 +108,6 @@ public class HdfsDirectory extends Directory { } } - public static void main(String[] args) { - System.out.println(new MetricName(HdfsDirectory.class,"name","scope")); - System.out.println(new MetricName("group","type","name","scope")); - } - /** * We keep the metrics separate per filesystem. */ @@ -126,10 +123,10 @@ public class HdfsDirectory extends Directory { if (metricsGroup == null) { String scope = uri.toString(); - Histogram readAccess = Metrics.newHistogram(new MetricName(ORG_APACHE_BLUR, BLUR, "Read Latency in \u00B5s", scope)); - Histogram writeAccess = Metrics.newHistogram(new MetricName(ORG_APACHE_BLUR, BLUR, "Write Latency in \u00B5s", scope)); - Meter readThroughput = Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, BLUR, "Read Throughput", scope), "Read Bytes", TimeUnit.SECONDS); - Meter writeThroughput = Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, BLUR, "Write Throughput", scope), "Write Bytes", TimeUnit.SECONDS); + Histogram readAccess = Metrics.newHistogram(new MetricName(ORG_APACHE_BLUR, HDFS, "Read Latency in \u00B5s", scope)); + Histogram writeAccess = Metrics.newHistogram(new MetricName(ORG_APACHE_BLUR, HDFS, "Write Latency in \u00B5s", scope)); + Meter readThroughput = Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, HDFS, "Read Throughput", scope), "Read Bytes", TimeUnit.SECONDS); + Meter writeThroughput = Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, HDFS, "Write Throughput", scope), "Write Bytes", TimeUnit.SECONDS); metricsGroup = new MetricsGroup(readAccess, writeAccess, readThroughput, writeThroughput); metricsGroupMap.put(uri, metricsGroup); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/cdbe70f6/src/blur-util/src/main/java/org/apache/blur/metrics/BlurMetrics.java ---------------------------------------------------------------------- diff --git a/src/blur-util/src/main/java/org/apache/blur/metrics/BlurMetrics.java b/src/blur-util/src/main/java/org/apache/blur/metrics/BlurMetrics.java deleted file mode 100644 index 16f1d11..0000000 --- a/src/blur-util/src/main/java/org/apache/blur/metrics/BlurMetrics.java +++ /dev/null @@ -1,129 +0,0 @@ -package org.apache.blur.metrics; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicLong; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.metrics.MetricsContext; -import org.apache.hadoop.metrics.MetricsRecord; -import org.apache.hadoop.metrics.MetricsUtil; -import org.apache.hadoop.metrics.Updater; -import org.apache.hadoop.metrics.jvm.JvmMetrics; - -public class BlurMetrics { - -} - -//public class BlurMetrics implements Updater { -// -// public static class MethodCall { -// public AtomicLong invokes = new AtomicLong(); -// public AtomicLong times = new AtomicLong(); -// } -// -// public AtomicLong blockCacheHit = new AtomicLong(0); -// public AtomicLong blockCacheMiss = new AtomicLong(0); -// public AtomicLong blockCacheEviction = new AtomicLong(0); -// public AtomicLong blockCacheSize = new AtomicLong(0); -// public AtomicLong rowReads = new AtomicLong(0); -// public AtomicLong rowWrites = new AtomicLong(0); -// public AtomicLong recordReads = new AtomicLong(0); -// public AtomicLong recordWrites = new AtomicLong(0); -// public AtomicLong queriesExternal = new AtomicLong(0); -// public AtomicLong queriesInternal = new AtomicLong(0); -// public AtomicLong blurShardBuffercacheAllocate1024 = new AtomicLong(0); -// public AtomicLong blurShardBuffercacheAllocate8192 = new AtomicLong(0); -// public AtomicLong blurShardBuffercacheAllocateOther = new AtomicLong(0); -// public AtomicLong blurShardBuffercacheLost = new AtomicLong(0); -// public Map<String, MethodCall> methodCalls = new ConcurrentHashMap<String, MethodCall>(); -// -// public AtomicLong tableCount = new AtomicLong(0); -// public AtomicLong rowCount = new AtomicLong(0); -// public AtomicLong recordCount = new AtomicLong(0); -// public AtomicLong indexCount = new AtomicLong(0); -// public AtomicLong indexMemoryUsage = new AtomicLong(0); -// public AtomicLong segmentCount = new AtomicLong(0); -// -// private MetricsRecord _metricsRecord; -// private long _previous = System.nanoTime(); -// -// public static void main(String[] args) throws InterruptedException { -// Configuration conf = new Configuration(); -// BlurMetrics blurMetrics = new BlurMetrics(conf); -// MethodCall methodCall = new MethodCall(); -// blurMetrics.methodCalls.put("test", methodCall); -// for (int i = 0; i < 100; i++) { -// blurMetrics.blockCacheHit.incrementAndGet(); -// blurMetrics.blockCacheMiss.incrementAndGet(); -// methodCall.invokes.incrementAndGet(); -// methodCall.times.addAndGet(56000000); -// Thread.sleep(500); -// } -// } -// -// public BlurMetrics(Configuration conf) { -// JvmMetrics.init("blur", Long.toString(System.currentTimeMillis())); -// MetricsContext metricsContext = MetricsUtil.getContext("blur"); -// _metricsRecord = MetricsUtil.createRecord(metricsContext, "metrics"); -// metricsContext.registerUpdater(this); -// } -// -// @Override -// public void doUpdates(MetricsContext context) { -// synchronized (this) { -// long now = System.nanoTime(); -// float seconds = (now - _previous) / 1000000000.0f; -// _metricsRecord.setMetric("blockcache.hit", getPerSecond(blockCacheHit.getAndSet(0), seconds)); -// _metricsRecord.setMetric("blockcache.miss", getPerSecond(blockCacheMiss.getAndSet(0), seconds)); -// _metricsRecord.setMetric("blockcache.eviction", getPerSecond(blockCacheEviction.getAndSet(0), seconds)); -// _metricsRecord.setMetric("blockcache.size", blockCacheSize.get()); -// _metricsRecord.setMetric("row.reads", getPerSecond(rowReads.getAndSet(0), seconds)); -// _metricsRecord.setMetric("row.writes", getPerSecond(rowWrites.getAndSet(0), seconds)); -// _metricsRecord.setMetric("record.reads", getPerSecond(recordReads.getAndSet(0), seconds)); -// _metricsRecord.setMetric("record.writes", getPerSecond(recordWrites.getAndSet(0), seconds)); -// _metricsRecord.setMetric("query.external", getPerSecond(queriesExternal.getAndSet(0), seconds)); -// _metricsRecord.setMetric("query.internal", getPerSecond(queriesInternal.getAndSet(0), seconds)); -// for (Entry<String, MethodCall> entry : methodCalls.entrySet()) { -// String key = entry.getKey(); -// MethodCall value = entry.getValue(); -// long invokes = value.invokes.getAndSet(0); -// long times = value.times.getAndSet(0); -// -// float avgTimes = (times / (float) invokes) / 1000000000.0f; -// _metricsRecord.setMetric("methodcalls." + key + ".count", getPerSecond(invokes, seconds)); -// _metricsRecord.setMetric("methodcalls." + key + ".time", avgTimes); -// } -// _metricsRecord.setMetric("tables", tableCount.get()); -// _metricsRecord.setMetric("rows", rowCount.get()); -// _metricsRecord.setMetric("records", recordCount.get()); -// _metricsRecord.setMetric("index.count", indexCount.get()); -// _metricsRecord.setMetric("index.memoryusage", indexMemoryUsage.get()); -// _metricsRecord.setMetric("index.segments", segmentCount.get()); -// _previous = now; -// } -// _metricsRecord.update(); -// } -// -// private float getPerSecond(long value, float seconds) { -// return (float) (value / seconds); -// } -// -//} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/cdbe70f6/src/blur-util/src/main/java/org/apache/blur/metrics/HeapMetrics.java ---------------------------------------------------------------------- diff --git a/src/blur-util/src/main/java/org/apache/blur/metrics/HeapMetrics.java b/src/blur-util/src/main/java/org/apache/blur/metrics/HeapMetrics.java deleted file mode 100644 index db286d4..0000000 --- a/src/blur-util/src/main/java/org/apache/blur/metrics/HeapMetrics.java +++ /dev/null @@ -1,107 +0,0 @@ -package org.apache.blur.metrics; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.PrintWriter; -import java.lang.management.ManagementFactory; -import java.lang.management.MemoryUsage; -import java.util.Timer; -import java.util.TimerTask; -import java.util.concurrent.TimeUnit; - -public class HeapMetrics { - -} - -//public class HeapMetrics extends TimerTask { -// -// private static final double ONE_GIG = 1024 * 1024 * 1024; -// private final int sampleSize = (int) TimeUnit.MINUTES.toSeconds(10); -// private final Timer timer; -// private final long period = TimeUnit.SECONDS.toMillis(1); -// private final long[] heapMemoryUsageUsedHistory = new long[sampleSize]; -// private final long[] heapMemoryUsageCommittedHistory = new long[sampleSize]; -// private final long[] timestamp = new long[sampleSize]; -// private volatile static HeapMetrics heapMetrics; -// private volatile int position = 0; -// -// public static synchronized HeapMetrics getInstance() { -// if (heapMetrics == null) { -// heapMetrics = new HeapMetrics(); -// } -// return heapMetrics; -// } -// -// private HeapMetrics() { -// timer = new Timer("HeapMetrics", true); -// timer.scheduleAtFixedRate(this, period, period); -// } -// -// @Override -// public void run() { -// synchronized (this) { -// if (position >= sampleSize) { -// position = 0; -// } -// MemoryUsage heapMemoryUsage = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage(); -// setValue(heapMemoryUsage.getUsed(), heapMemoryUsageUsedHistory); -// setValue(heapMemoryUsage.getCommitted(), heapMemoryUsageCommittedHistory); -// setValue(System.currentTimeMillis(), timestamp); -// position++; -// } -// } -// -// private void setValue(long value, long[] history) { -// history[position] = value; -// } -// -// public void writeJson(PrintWriter out) { -// synchronized (this) { -// out.print("{\"labels\":["); -// out.print("{\"name\":\"used\",\"style\":{\"stroke\":\"RoyalBlue\"}},"); -// out.print("{\"name\":\"committed\",\"style\":{\"stroke\":\"Red\"}}"); -// out.print("],\"data\":["); -// int p = position; -// boolean comma = false; -// for (int i = 0; i < sampleSize; i++, p++) { -// if (p >= sampleSize) { -// p = 0; -// } -// double used = ((double) heapMemoryUsageUsedHistory[p]) / ONE_GIG; -// double committed = ((double) heapMemoryUsageCommittedHistory[p]) / ONE_GIG; -// -// long t = timestamp[p]; -// if (t == 0) { -// continue; -// } -// if (comma) { -// out.print(","); -// } -// out.print("{\"used\":"); -// out.print(used); -// out.print(",\"committed\":"); -// out.print(committed); -// out.print(",\"recordTime\":"); -// out.print(t); -// out.print('}'); -// comma = true; -// } -// out.print("]}"); -// } -// } -//} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/cdbe70f6/src/blur-util/src/main/java/org/apache/blur/metrics/MetricsConstants.java ---------------------------------------------------------------------- diff --git a/src/blur-util/src/main/java/org/apache/blur/metrics/MetricsConstants.java b/src/blur-util/src/main/java/org/apache/blur/metrics/MetricsConstants.java index 114fafe..1d0ee9c 100644 --- a/src/blur-util/src/main/java/org/apache/blur/metrics/MetricsConstants.java +++ b/src/blur-util/src/main/java/org/apache/blur/metrics/MetricsConstants.java @@ -16,5 +16,6 @@ public class MetricsConstants { public static final String ADD = "add"; public static final String SEARCH = "search"; public static final String REMOTE = "remote"; + public static final String HDFS = "HDFS"; public static final String LOCAL = "local"; } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/cdbe70f6/src/blur-util/src/main/java/org/apache/blur/metrics/QueryMetrics.java ---------------------------------------------------------------------- diff --git a/src/blur-util/src/main/java/org/apache/blur/metrics/QueryMetrics.java b/src/blur-util/src/main/java/org/apache/blur/metrics/QueryMetrics.java deleted file mode 100644 index 293e8de..0000000 --- a/src/blur-util/src/main/java/org/apache/blur/metrics/QueryMetrics.java +++ /dev/null @@ -1,199 +0,0 @@ -package org.apache.blur.metrics; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.PrintWriter; -import java.util.Timer; -import java.util.TimerTask; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; - -public class QueryMetrics {} - -//public class QueryMetrics extends TimerTask { -// -// private final int sampleSize = (int) TimeUnit.MINUTES.toSeconds(10); -// private final AtomicLong _queryCount = new AtomicLong(); -// private final AtomicLong _queryTime = new AtomicLong(); -// private final AtomicLong _queryExceptionCount = new AtomicLong(); -// private final AtomicLong _dataFetchCount = new AtomicLong(); -// private final AtomicLong _dataFetchTime = new AtomicLong(); -// private final AtomicLong _dataFetchRecordCount = new AtomicLong(); -// private final AtomicLong _dataMutateCount = new AtomicLong(); -// private final AtomicLong _dataMutateTime = new AtomicLong(); -// private final AtomicLong _dataMutateRecordCount = new AtomicLong(); -// private final Timer timer; -// private final long period = TimeUnit.SECONDS.toMillis(1); -// private final double[] _queryRate = new double[sampleSize]; -// private final double[] _queryResponse = new double[sampleSize]; -// private final double[] _queryExceptionRate = new double[sampleSize]; -// private final double[] _fetchRate = new double[sampleSize]; -// private final double[] _fetchResponse = new double[sampleSize]; -// private final double[] _fetchRecordRate = new double[sampleSize]; -// private final double[] _mutateRate = new double[sampleSize]; -// private final double[] _mutateResponse = new double[sampleSize]; -// private final double[] _mutateRecordRate = new double[sampleSize]; -// private final long[] timestamp = new long[sampleSize]; -// private volatile static QueryMetrics instance; -// private volatile int position; -// -// public static synchronized QueryMetrics getInstance() { -// if (instance == null) { -// instance = new QueryMetrics(); -// } -// return instance; -// } -// -// private QueryMetrics() { -// timer = new Timer("QueryMetrics", true); -// timer.scheduleAtFixedRate(this, period, period); -// } -// -// @Override -// public void run() { -// synchronized (this) { -// if (position >= sampleSize) { -// position = 0; -// } -// long queryCount = _queryCount.getAndSet(0); -// long queryTime = _queryTime.getAndSet(0); -// long queryExceptionCount = _queryExceptionCount.getAndSet(0); -// -// long dataFetchCount = _dataFetchCount.getAndSet(0); -// long dataFetchTime = _dataFetchTime.getAndSet(0); -// long dataFetchRecordCount = _dataFetchRecordCount.getAndSet(0); -// -// long dataMutateCount = _dataMutateCount.getAndSet(0); -// long dataMutateTime = _dataMutateTime.getAndSet(0); -// long dataMutateRecordCount = _dataMutateRecordCount.getAndSet(0); -// -// _queryRate[position] = queryCount; -// _queryResponse[position] = TimeUnit.NANOSECONDS.toMillis(queryTime) / (double) queryCount; -// -// _queryExceptionRate[position] = queryExceptionCount; -// -// _fetchRate[position] = dataFetchCount; -// _fetchResponse[position] = TimeUnit.NANOSECONDS.toMillis(dataFetchTime) / (double) dataFetchCount; -// _fetchRecordRate[position] = dataFetchRecordCount; -// -// _mutateRate[position] = dataMutateCount; -// _mutateResponse[position] = TimeUnit.NANOSECONDS.toMillis(dataMutateTime) / (double) dataMutateCount; -// _mutateRecordRate[position] = dataMutateRecordCount; -// -// timestamp[position] = System.currentTimeMillis(); -// position++; -// } -// } -// -// public void recordQuery(long totalTimeNs) { -// _queryCount.incrementAndGet(); -// _queryTime.addAndGet(totalTimeNs); -// } -// -// public void recordQueryExceptions() { -// _queryExceptionCount.incrementAndGet(); -// } -// -// public void recordDataFetch(long totalTimeNs, long records) { -// _dataFetchCount.incrementAndGet(); -// _dataFetchTime.addAndGet(totalTimeNs); -// _dataFetchRecordCount.addAndGet(records); -// } -// -// public void recordDataMutate(long totalTimeNs, long records) { -// _dataMutateCount.incrementAndGet(); -// _dataMutateTime.addAndGet(totalTimeNs); -// _dataMutateRecordCount.addAndGet(records); -// } -// -// public void writeGraph1Json(PrintWriter out) { -// synchronized (this) { -// out.print("{\"labels\":["); -// out.print("{\"name\":\"query\",\"style\":{\"stroke\":\"RoyalBlue\"}},"); -// out.print("{\"name\":\"fetch\",\"style\":{\"stroke\":\"Black\"}},"); -// out.print("{\"name\":\"mutate\",\"style\":{\"stroke\":\"Orange\"}},"); -// out.print("{\"name\":\"except\",\"style\":{\"stroke\":\"Red\"}}"); -// out.print("],\"data\":["); -// int p = position; -// boolean comma = false; -// for (int i = 0; i < sampleSize; i++, p++) { -// if (p >= sampleSize) { -// p = 0; -// } -// double query = _queryRate[p]; -// double fetch = _fetchRate[p]; -// double mutate = _mutateRate[p]; -// double except = _queryExceptionRate[p]; -// long t = timestamp[p]; -// if (t == 0) { -// continue; -// } -// if (comma) { -// out.print(","); -// } -// out.print("{\"query\":"); -// out.print(query); -// out.print(",\"fetch\":"); -// out.print(fetch); -// out.print(",\"mutate\":"); -// out.print(mutate); -// out.print(",\"except\":"); -// out.print(except); -// out.print(",\"recordTime\":"); -// out.print(t); -// out.print('}'); -// comma = true; -// } -// out.print("]}"); -// } -// } -// -// public void writeGraph2Json(PrintWriter out) { -// synchronized (this) { -// out.print("{\"labels\":["); -// out.print("{\"name\":\"fetchRecord\",\"style\":{\"stroke\":\"Black\"}},"); -// out.print("{\"name\":\"mutateRecord\",\"style\":{\"stroke\":\"Orange\"}}"); -// out.print("],\"data\":["); -// int p = position; -// boolean comma = false; -// for (int i = 0; i < sampleSize; i++, p++) { -// if (p >= sampleSize) { -// p = 0; -// } -// double fetch = _fetchRecordRate[p]; -// double mutate = _mutateRecordRate[p]; -// long t = timestamp[p]; -// if (t == 0) { -// continue; -// } -// if (comma) { -// out.print(","); -// } -// out.print("{\"fetchRecord\":"); -// out.print(fetch); -// out.print(",\"mutateRecord\":"); -// out.print(mutate); -// out.print(",\"recordTime\":"); -// out.print(t); -// out.print('}'); -// comma = true; -// } -// out.print("]}"); -// } -// } -//} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/cdbe70f6/src/blur-util/src/main/java/org/apache/blur/metrics/SystemLoadMetrics.java ---------------------------------------------------------------------- diff --git a/src/blur-util/src/main/java/org/apache/blur/metrics/SystemLoadMetrics.java b/src/blur-util/src/main/java/org/apache/blur/metrics/SystemLoadMetrics.java deleted file mode 100644 index a3a701b..0000000 --- a/src/blur-util/src/main/java/org/apache/blur/metrics/SystemLoadMetrics.java +++ /dev/null @@ -1,89 +0,0 @@ -package org.apache.blur.metrics; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.PrintWriter; -import java.lang.management.ManagementFactory; -import java.lang.management.OperatingSystemMXBean; -import java.util.Timer; -import java.util.TimerTask; -import java.util.concurrent.TimeUnit; - -public class SystemLoadMetrics extends TimerTask { - - private final int sampleSize = (int) TimeUnit.MINUTES.toSeconds(10); - private final Timer timer; - private final long period = TimeUnit.SECONDS.toMillis(1); - private final double[] systemLoadAverageHistory = new double[sampleSize]; - private final long[] timestamp = new long[sampleSize]; - private volatile static SystemLoadMetrics instance; - private volatile int position = 0; - - public static synchronized SystemLoadMetrics getInstance() { - if (instance == null) { - instance = new SystemLoadMetrics(); - } - return instance; - } - - private SystemLoadMetrics() { - timer = new Timer("SystemLoadMetrics", true); - timer.scheduleAtFixedRate(this, period, period); - } - - @Override - public void run() { - synchronized (this) { - if (position >= sampleSize) { - position = 0; - } - OperatingSystemMXBean operatingSystemMXBean = ManagementFactory.getOperatingSystemMXBean(); - systemLoadAverageHistory[position] = operatingSystemMXBean.getSystemLoadAverage(); - timestamp[position] = System.currentTimeMillis(); - position++; - } - } - - public void writeJson(PrintWriter out) { - synchronized (this) { - out.print("{\"labels\":[{\"name\":\"load\",\"style\":{\"stroke\":\"Red\"}}],\"data\":["); - int p = position; - boolean comma = false; - for (int i = 0; i < sampleSize; i++, p++) { - if (p >= sampleSize) { - p = 0; - } - double load = systemLoadAverageHistory[p]; - long t = timestamp[p]; - if (t == 0) { - continue; - } - if (comma) { - out.print(","); - } - out.print("{\"load\":"); - out.print(load); - out.print(",\"recordTime\":"); - out.print(t); - out.print('}'); - comma = true; - } - out.print("]}"); - } - } -}
