Updated Branches: refs/heads/master 6dd9aa3f7 -> ce716b27f
Added back in the hap usage metrics. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/ce716b27 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/ce716b27 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/ce716b27 Branch: refs/heads/master Commit: ce716b27f786669eb9155cbbc6d14a2d0a97b8de Parents: 6dd9aa3 Author: Aaron McCurry <[email protected]> Authored: Wed Jun 19 20:39:05 2013 -0400 Committer: Aaron McCurry <[email protected]> Committed: Wed Jun 19 20:39:05 2013 -0400 ---------------------------------------------------------------------- .../indexserver/DistributedIndexServer.java | 41 +++++++++++--------- .../java/org/apache/blur/utils/BlurUtil.java | 6 --- .../java/org/apache/blur/utils/ObjectSize.java | 37 ------------------ .../org/apache/blur/utils/BlurUtilsTest.java | 8 ---- 4 files changed, 22 insertions(+), 70 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ce716b27/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java index 215d1e6..85c1810 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java +++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java @@ -51,7 +51,6 @@ import org.apache.blur.log.LogFactory; import org.apache.blur.lucene.search.FairSimilarity; import org.apache.blur.lucene.store.refcounter.DirectoryReferenceFileGC; import org.apache.blur.lucene.store.refcounter.IndexInputCloser; -import org.apache.blur.lucene.warmup.TraceableDirectory; import org.apache.blur.manager.BlurFilterCache; import org.apache.blur.manager.clusterstatus.ClusterStatus; import org.apache.blur.manager.clusterstatus.ZookeeperPathConstants; @@ -81,8 +80,10 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.RamUsageEstimator; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.ZooDefs.Ids; @@ -94,7 +95,7 @@ import com.yammer.metrics.core.MetricName; public class DistributedIndexServer extends AbstractIndexServer { private static final Log LOG = LogFactory.getLog(DistributedIndexServer.class); - private static final long _delay = TimeUnit.SECONDS.toMillis(5); + private static final long _delay = TimeUnit.SECONDS.toMillis(10); private Map<String, BlurAnalyzer> _tableAnalyzers = new ConcurrentHashMap<String, BlurAnalyzer>(); private Map<String, TableDescriptor> _tableDescriptors = new ConcurrentHashMap<String, TableDescriptor>(); @@ -144,7 +145,7 @@ public class DistributedIndexServer extends AbstractIndexServer { MetricName indexCount = new MetricName(ORG_APACHE_BLUR, BLUR, INDEX_COUNT, _cluster); MetricName segmentCount = new MetricName(ORG_APACHE_BLUR, BLUR, SEGMENT_COUNT, _cluster); MetricName indexMemoryUsage = new MetricName(ORG_APACHE_BLUR, BLUR, INDEX_MEMORY_USAGE, _cluster); - + Metrics.newGauge(tableCount, new AtomicLongGauge(_tableCount)); Metrics.newGauge(indexCount, new AtomicLongGauge(_indexCount)); Metrics.newGauge(segmentCount, new AtomicLongGauge(_segmentCount)); @@ -165,11 +166,12 @@ public class DistributedIndexServer extends AbstractIndexServer { _indexCloser = new BlurIndexCloser(); _indexCloser.init(); setupFlushCacheTimer(); - + registerMyselfAsMemberOfCluster(); String onlineShardsPath = ZookeeperPathConstants.getOnlineShardsPath(_cluster); String safemodePath = ZookeeperPathConstants.getSafemodePath(_cluster); - SafeMode safeMode = new SafeMode(_zookeeper, safemodePath, onlineShardsPath, TimeUnit.MILLISECONDS, _safeModeDelay, TimeUnit.SECONDS, 60); + SafeMode safeMode = new SafeMode(_zookeeper, safemodePath, onlineShardsPath, TimeUnit.MILLISECONDS, _safeModeDelay, + TimeUnit.SECONDS, 60); safeMode.registerNode(getNodeName(), BlurUtil.getVersion().getBytes()); _running.set(true); @@ -250,18 +252,19 @@ public class DistributedIndexServer extends AbstractIndexServer { private void updateMetrics(Map<String, BlurIndex> indexes, AtomicLong segmentCount, AtomicLong indexMemoryUsage) throws IOException { // @TODO not sure how to do this yet - // for (BlurIndex index : indexes.values()) { - // IndexReader reader = index.getIndexReader(); - // try { - // IndexReader[] readers = reader.getSequentialSubReaders(); - // if (readers != null) { - // segmentCount.addAndGet(readers.length); - // } - // indexMemoryUsage.addAndGet(BlurUtil.getMemoryUsage(reader)); - // } finally { - // reader.decRef(); - // } - // } + + indexMemoryUsage.addAndGet(RamUsageEstimator.sizeOf(indexes)); + for (BlurIndex index : indexes.values()) { + IndexSearcherClosable indexSearcherClosable = index.getIndexReader(); + try { + IndexReader indexReader = indexSearcherClosable.getIndexReader(); + IndexReaderContext context = indexReader.getContext(); + segmentCount.addAndGet(context.leaves().size()); + + } finally { + indexSearcherClosable.close(); + } + } } }, _delay, _delay); } @@ -369,7 +372,7 @@ public class DistributedIndexServer extends AbstractIndexServer { @Override public void close() { if (_running.get()) { - + _shardStateManager.close(); _running.set(false); closeAllIndexes(); @@ -464,7 +467,7 @@ public class DistributedIndexServer extends AbstractIndexServer { } else { dir = directory; } - + BlurIndex index; if (_clusterStatus.isReadOnly(true, _cluster, table)) { BlurIndexReader reader = new BlurIndexReader(shardContext, dir, _refresher, _indexCloser); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ce716b27/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java b/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java index 30efdce..ef386c2 100644 --- a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java +++ b/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java @@ -89,7 +89,6 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.util.RamUsageEstimator; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.KeeperException.Code; @@ -423,11 +422,6 @@ public class BlurUtil { return seconds / TimeUnit.HOURS.toSeconds(1); } - public static long getMemoryUsage(IndexReader r) { - long sizeOf = RamUsageEstimator.sizeOf(r); - return sizeOf; - } - public static void createPath(ZooKeeper zookeeper, String path, byte[] data) throws KeeperException, InterruptedException { zookeeper.create(path, data, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ce716b27/blur-core/src/main/java/org/apache/blur/utils/ObjectSize.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/utils/ObjectSize.java b/blur-core/src/main/java/org/apache/blur/utils/ObjectSize.java deleted file mode 100644 index 2c9a2c9..0000000 --- a/blur-core/src/main/java/org/apache/blur/utils/ObjectSize.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.apache.blur.utils; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import java.lang.instrument.Instrumentation; - -public class ObjectSize { - - private static Instrumentation instrumentation; - - public static void premain(String agentArgs, Instrumentation inst) { - instrumentation = inst; - } - - public static long getSizeInBytes(Object o) { - if (instrumentation == null) - return -1; - if (o == null) - return 0; - return instrumentation.getObjectSize(o); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ce716b27/blur-core/src/test/java/org/apache/blur/utils/BlurUtilsTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/utils/BlurUtilsTest.java b/blur-core/src/test/java/org/apache/blur/utils/BlurUtilsTest.java index 4923862..5346f6d 100644 --- a/blur-core/src/test/java/org/apache/blur/utils/BlurUtilsTest.java +++ b/blur-core/src/test/java/org/apache/blur/utils/BlurUtilsTest.java @@ -19,7 +19,6 @@ package org.apache.blur.utils; import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.File; @@ -92,13 +91,6 @@ public class BlurUtilsTest { } @Test - public void testMemoryUsage() throws CorruptIndexException, LockObtainFailedException, IOException { - IndexReader reader = getReader(); - long memoryUsage = BlurUtil.getMemoryUsage(reader); - assertTrue(memoryUsage > 0); - } - - @Test public void testValidateShardCount() throws IOException { File file = new File(TMPDIR, "ValidateShardCount-test"); rm(file);
