Updated Branches: refs/heads/0.2-dev 1a8ebf48c -> 0270f9a09
Adding new metrics library, plus some new metrics. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/0270f9a0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/0270f9a0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/0270f9a0 Branch: refs/heads/0.2-dev Commit: 0270f9a09f314e51d6b2df5d69f4dab10512b9e5 Parents: 1a8ebf4 Author: Aaron McCurry <[email protected]> Authored: Tue Mar 5 23:30:40 2013 -0500 Committer: Aaron McCurry <[email protected]> Committed: Tue Mar 5 23:30:40 2013 -0500 ---------------------------------------------------------------------- .../java/org/apache/blur/server/BlurServer.java | 30 ++++++++++++ .../store/refcounter/DirectoryReferenceFileGC.java | 14 ++++++ .../org/apache/blur/store/hdfs/HdfsDirectory.java | 26 ++++++++++- src/blur-util/pom.xml | 36 +++++++-------- src/pom.xml | 1 + 5 files changed, 86 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0270f9a0/src/blur-core/src/main/java/org/apache/blur/server/BlurServer.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/server/BlurServer.java b/src/blur-core/src/main/java/org/apache/blur/server/BlurServer.java index e6185ea..9dfda86 100644 --- a/src/blur-core/src/main/java/org/apache/blur/server/BlurServer.java +++ b/src/blur-core/src/main/java/org/apache/blur/server/BlurServer.java @@ -74,6 +74,9 @@ import org.apache.blur.utils.ThriftLuceneConversion; import org.apache.lucene.search.IndexSearcher; import org.apache.thrift.TException; +import com.yammer.metrics.Metrics; +import com.yammer.metrics.core.Meter; + public class BlurServer extends TableAdmin implements Iface { private static final Log LOG = LogFactory.getLog(BlurServer.class); @@ -88,6 +91,14 @@ public class BlurServer extends TableAdmin implements Iface { private SessionManager _sessionManager; private String _nodeName; private QueryStatusContainer _queryStatusContainer; + private Meter localSearches; + private Meter remoteSearches; + private Meter localAddRequests; + private Meter remoteAddRequests; + private Meter localUpdateRequests; + private Meter remoteUpdateRequests; + private Meter localDeleteRequests; + private Meter remoteDeleteRequests; static class SearchAction { enum ACTION { @@ -126,6 +137,15 @@ public class BlurServer extends TableAdmin implements Iface { _queryStatusContainer = new QueryStatusContainer(_configuration.getInt(BLUR_QUERYSTATUS_TRACKING_MAX, 16384)); _sessionManager = new SessionManager(); _sessionManager.init(); + + localSearches = Metrics.newMeter(BlurServer.class, "searches", "local", TimeUnit.SECONDS); + remoteSearches = Metrics.newMeter(BlurServer.class, "searches", "remote", TimeUnit.SECONDS); + localAddRequests = Metrics.newMeter(BlurServer.class, "addRequests", "local", TimeUnit.SECONDS); + remoteAddRequests = Metrics.newMeter(BlurServer.class, "addRequests", "remote", TimeUnit.SECONDS); + localUpdateRequests = Metrics.newMeter(BlurServer.class, "updateRequests", "local", TimeUnit.SECONDS); + remoteUpdateRequests = Metrics.newMeter(BlurServer.class, "updateRequests", "remote", TimeUnit.SECONDS); + localDeleteRequests = Metrics.newMeter(BlurServer.class, "deleteRequests", "local", TimeUnit.SECONDS); + remoteDeleteRequests = Metrics.newMeter(BlurServer.class, "deleteRequests", "remote", TimeUnit.SECONDS); } @Override @@ -219,11 +239,13 @@ public class BlurServer extends TableAdmin implements Iface { final int shardIndex = action.shardIndex; if (action.type == ACTION.LOCAL) { searchCallable = new LocalSearchCallable(shardIndex, action.indexSearcher, after, query, filter, sort, numberToFetch, doDocScores, doMaxScore); + localSearches.mark(); future = _indexSearcherExecutor.submit(searchCallable); } else if (action.type == ACTION.REMOTE) { // @TODO need to send only one call per server, instead of one for // each shard in the table searchCallable = new RemoteSearchCallable(session, action.remoteServer, shardIndex, queryArgs); + remoteSearches.mark(); future = _indexSearcherExecutor.submit(searchCallable); } else { throw new BlurException("Unsupported action of [" + action.type + "]", null); @@ -343,8 +365,10 @@ public class BlurServer extends TableAdmin implements Iface { BlurIndex index = getIndex(table, shardIndex); List<Document> indexableDocuments = BlurValidations.getAllIndexableDocuments(documents); if (index == null) { + remoteAddRequests.mark(documents.size()); generations.addAll(forwardAddDocuments(options, indexableDocuments)); } else { + localAddRequests.mark(documents.size()); long generation = index.addDocuments(waitToBeVisible, writeAheadLog, indexableDocuments); generations.add(new Generation(table, shardIndex, generation)); } @@ -404,8 +428,10 @@ public class BlurServer extends TableAdmin implements Iface { try { BlurIndex index = getIndex(table, shardIndex); if (index == null) { + remoteDeleteRequests.mark(queries.size()); generations.addAll(forwardDeleteDocumentsByQueries(options, queries)); } else { + localDeleteRequests.mark(queries.size()); long generation = index.deleteDocuments(waitToBeVisible, writeAheadLog, queries.toArray(new Query[queries.size()])); generations.add(new Generation(table, shardIndex, generation)); } @@ -426,8 +452,10 @@ public class BlurServer extends TableAdmin implements Iface { try { BlurIndex index = getIndex(table, shardIndex); if (index == null) { + remoteDeleteRequests.mark(terms.size()); generations.addAll(forwardDeleteDocuments(options, terms)); } else { + localDeleteRequests.mark(terms.size()); long generation = index.deleteDocuments(waitToBeVisible, writeAheadLog, terms.toArray(new Term[terms.size()])); generations.add(new Generation(table, shardIndex, generation)); } @@ -449,8 +477,10 @@ public class BlurServer extends TableAdmin implements Iface { BlurIndex index = getIndex(table, shardIndex); List<UpdatePackage> updatePackagesWithIndexableDocs = BlurValidations.getAllIndexablePackages(updatePackages); if (index == null) { + remoteUpdateRequests.mark(updatePackages.size()); generations.addAll(forwardUpdateDocuments(options, updatePackagesWithIndexableDocs)); } else { + localUpdateRequests.mark(updatePackages.size()); long generation = index.updateDocuments(waitToBeVisible, writeAheadLog, updatePackagesWithIndexableDocs); generations.add(new Generation(table, shardIndex, generation)); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0270f9a0/src/blur-store/src/main/java/org/apache/blur/lucene/store/refcounter/DirectoryReferenceFileGC.java ---------------------------------------------------------------------- diff --git a/src/blur-store/src/main/java/org/apache/blur/lucene/store/refcounter/DirectoryReferenceFileGC.java b/src/blur-store/src/main/java/org/apache/blur/lucene/store/refcounter/DirectoryReferenceFileGC.java index 9db6256..b8a9301 100644 --- a/src/blur-store/src/main/java/org/apache/blur/lucene/store/refcounter/DirectoryReferenceFileGC.java +++ b/src/blur-store/src/main/java/org/apache/blur/lucene/store/refcounter/DirectoryReferenceFileGC.java @@ -28,6 +28,9 @@ import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; import org.apache.lucene.store.Directory; +import com.yammer.metrics.Metrics; +import com.yammer.metrics.core.Gauge; + public class DirectoryReferenceFileGC extends TimerTask { private static final Log LOG = LogFactory.getLog(DirectoryReferenceFileGC.class); @@ -35,6 +38,7 @@ public class DirectoryReferenceFileGC extends TimerTask { private Timer _timer; private long _delay = 5000; private LinkedBlockingQueue<Value> _queue; + private volatile int numberOfFilesToBeDeleted = 0; public static class Value { public Value(Directory directory, String name, Map<String, AtomicInteger> refs) { @@ -65,6 +69,12 @@ public class DirectoryReferenceFileGC extends TimerTask { _timer = new Timer("Blur-File-GC", true); _timer.scheduleAtFixedRate(this, _delay, _delay); _queue = new LinkedBlockingQueue<Value>(); + Metrics.newGauge(getClass(), "filesToBeDeleted", new Gauge<Integer>() { + @Override + public Integer value() { + return numberOfFilesToBeDeleted; + } + }); } public void add(Directory directory, String name, Map<String, AtomicInteger> refs) { @@ -83,15 +93,19 @@ public class DirectoryReferenceFileGC extends TimerTask { @Override public void run() { Iterator<Value> iterator = _queue.iterator(); + int count = 0; while (iterator.hasNext()) { Value value = iterator.next(); try { if (value.tryToDelete()) { iterator.remove(); + } else { + count++; } } catch (IOException e) { LOG.error("Unknown error", e); } } + numberOfFilesToBeDeleted = count; } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0270f9a0/src/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java ---------------------------------------------------------------------- diff --git a/src/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java b/src/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java index 18de3e7..fdc5a52 100644 --- a/src/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java +++ b/src/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java @@ -20,6 +20,8 @@ package org.apache.blur.store.hdfs; import java.io.FileNotFoundException; import java.io.IOException; import java.util.Collection; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; import org.apache.blur.log.Log; @@ -40,6 +42,9 @@ import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.NoLockFactory; +import com.yammer.metrics.Metrics; +import com.yammer.metrics.core.Histogram; + public class HdfsDirectory extends Directory { private static final Log LOG = LogFactory.getLog(HdfsDirectory.class); @@ -80,10 +85,22 @@ public class HdfsDirectory extends Directory { private final Path path; private final FileSystem fileSystem; + private final Histogram readAccess; + private static Map<String, Histogram> readAccessMap = new ConcurrentHashMap<String, Histogram>(); + public HdfsDirectory(Configuration configuration, Path path) throws IOException { this.path = path; fileSystem = path.getFileSystem(configuration); setLockFactory(NoLockFactory.getNoLockFactory()); + synchronized (readAccessMap) { + Histogram histogram = readAccessMap.get(fileSystem.toString()); + if (histogram == null) { + histogram = Metrics.newHistogram(HdfsDirectory.class, "Read Access [" + fileSystem.getUri() + "]"); + readAccessMap.put(fileSystem.toString(), histogram); + } + readAccess = histogram; + } + } @Override @@ -96,12 +113,14 @@ public class HdfsDirectory extends Directory { private final long len; private FSDataInputStream inputStream; private boolean isClone; + private final Histogram readAccess; - public HdfsIndexInput(FileSystem fileSystem, Path filePath) throws IOException { + public HdfsIndexInput(FileSystem fileSystem, Path filePath, Histogram readAccess) throws IOException { super(filePath.toString()); inputStream = fileSystem.open(filePath); FileStatus fileStatus = fileSystem.getFileStatus(filePath); len = fileStatus.getLen(); + this.readAccess = readAccess; } @Override @@ -117,8 +136,11 @@ public class HdfsDirectory extends Directory { @Override protected void readInternal(byte[] b, int offset, int length) throws IOException { synchronized (inputStream) { + long start = System.nanoTime(); inputStream.seek(getFilePointer()); inputStream.readFully(b, offset, length); + long end = System.nanoTime(); + readAccess.update((end - start) / 1000); } } @@ -175,7 +197,7 @@ public class HdfsDirectory extends Directory { throw new FileNotFoundException("File [" + name + "] not found."); } Path filePath = getPath(name); - return new HdfsIndexInput(fileSystem, filePath); + return new HdfsIndexInput(fileSystem, filePath, readAccess); } @Override http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0270f9a0/src/blur-util/pom.xml ---------------------------------------------------------------------- diff --git a/src/blur-util/pom.xml b/src/blur-util/pom.xml index feeb135..5af96d2 100644 --- a/src/blur-util/pom.xml +++ b/src/blur-util/pom.xml @@ -1,22 +1,14 @@ <?xml version="1.0" encoding="UTF-8" ?> -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---> +<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor + license agreements. See the NOTICE file distributed with this work for additional + information regarding copyright ownership. The ASF licenses this file to + you under the Apache License, Version 2.0 (the "License"); you may not use + this file except in compliance with the License. You may obtain a copy of + the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required + by applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS + OF ANY KIND, either express or implied. See the License for the specific + language governing permissions and limitations under the License. --> <project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> <modelVersion>4.0.0</modelVersion> @@ -24,7 +16,7 @@ under the License. <groupId>org.apache.blur</groupId> <artifactId>blur</artifactId> <version>0.2.0-SNAPSHOT</version> - <relativePath>../pom.xml</relativePath> + <relativePath>../pom.xml</relativePath> </parent> <groupId>org.apache.blur</groupId> <artifactId>blur-util</artifactId> @@ -103,8 +95,14 @@ under the License. </exclusion> </exclusions> </dependency> + <dependency> + <groupId>com.yammer.metrics</groupId> + <artifactId>metrics-core</artifactId> + <version>${metrics-core.version}</version> + </dependency> </dependencies> + <repositories> <repository> <id>libdir</id> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0270f9a0/src/pom.xml ---------------------------------------------------------------------- diff --git a/src/pom.xml b/src/pom.xml index 833d404..5370805 100644 --- a/src/pom.xml +++ b/src/pom.xml @@ -50,6 +50,7 @@ under the License. <concurrentlinkedhashmap-lru.version>1.3.2</concurrentlinkedhashmap-lru.version> <jline.version>2.7</jline.version> <guava.version>12.0</guava.version> + <metrics-core.version>2.2.0</metrics-core.version> </properties> <modules>
