Updated Branches: refs/heads/master 2f18b3592 -> 001f78878
Refactoring some code, and trying to make the blur cluster test more stable. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/e1a7fe6c Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/e1a7fe6c Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/e1a7fe6c Branch: refs/heads/master Commit: e1a7fe6cab655f5cfc6eb7f52c0a174697553dda Parents: 0e91346 Author: Aaron McCurry <[email protected]> Authored: Thu Nov 21 07:45:19 2013 -0500 Committer: Aaron McCurry <[email protected]> Committed: Thu Nov 21 07:48:08 2013 -0500 ---------------------------------------------------------------------- .../org/apache/blur/manager/IndexManager.java | 10 ++-- .../indexserver/DistributedIndexServer.java | 12 +++-- .../apache/blur/manager/writer/BlurIndex.java | 57 ++++++++++---------- .../blur/manager/writer/BlurIndexReadOnly.java | 4 +- .../blur/manager/writer/BlurIndexReader.java | 2 +- .../blur/manager/writer/BlurNRTIndex.java | 2 +- .../manager/writer/TransactionRecorder.java | 2 +- .../manager/writer/BlurIndexReaderTest.java | 4 +- .../blur/manager/writer/BlurNRTIndexTest.java | 8 +-- .../org/apache/blur/thrift/BlurClusterTest.java | 2 + 10 files changed, 55 insertions(+), 48 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e1a7fe6c/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java index 1d833dd..0b0be0a 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java +++ b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java @@ -275,7 +275,7 @@ public class IndexManager { searcher = shardServerContext.getIndexSearcherClosable(table, shard); } if (searcher == null) { - searcher = index.getIndexReader(); + searcher = index.getIndexSearcher(); usedCache = false; } @@ -340,7 +340,7 @@ public class IndexManager { if (blurIndex == null) { throw new BException("Shard [" + shardName + "] is not being servered by this shardserver."); } - IndexSearcherClosable searcher = blurIndex.getIndexReader(); + IndexSearcherClosable searcher = blurIndex.getIndexSearcher(); try { BooleanQuery query = new BooleanQuery(); if (selector.recordOnly) { @@ -692,7 +692,7 @@ public class IndexManager { @Override public Long call(Entry<String, BlurIndex> input) throws Exception { BlurIndex index = input.getValue(); - IndexSearcherClosable searcher = index.getIndexReader(); + IndexSearcherClosable searcher = index.getIndexSearcher(); try { return recordFrequency(searcher.getIndexReader(), columnFamily, columnName, value); } finally { @@ -728,7 +728,7 @@ public class IndexManager { @Override public List<String> call(Entry<String, BlurIndex> input) throws Exception { BlurIndex index = input.getValue(); - IndexSearcherClosable searcher = index.getIndexReader(); + IndexSearcherClosable searcher = index.getIndexSearcher(); try { return terms(searcher.getIndexReader(), columnFamily, columnName, startWith, size); } finally { @@ -1119,7 +1119,7 @@ public class IndexManager { String shard = entry.getKey(); _status.attachThread(shard); BlurIndex index = entry.getValue(); - IndexSearcherClosable searcher = index.getIndexReader(); + IndexSearcherClosable searcher = index.getIndexSearcher(); try { IndexReader indexReader = searcher.getIndexReader(); if (indexReader instanceof ExitableReader) { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e1a7fe6c/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java index fa38de6..80b5e0d 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java +++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java @@ -46,6 +46,7 @@ import org.apache.blur.manager.clusterstatus.ClusterStatus; import org.apache.blur.manager.clusterstatus.ZookeeperPathConstants; import org.apache.blur.manager.writer.BlurIndex; import org.apache.blur.manager.writer.BlurIndexCloser; +import org.apache.blur.manager.writer.BlurIndexNRTSimple; import org.apache.blur.manager.writer.BlurIndexReadOnly; import org.apache.blur.manager.writer.BlurIndexRefresher; import org.apache.blur.manager.writer.BlurNRTIndex; @@ -453,7 +454,7 @@ public class DistributedIndexServer extends AbstractDistributedIndexServer { ShardContext shardContext = ShardContext.create(tableContext, shard); Directory dir; - + TableDescriptor descriptor = tableContext.getDescriptor(); boolean blockCacheEnabled = descriptor.isBlockCaching(); if (blockCacheEnabled) { @@ -464,8 +465,13 @@ public class DistributedIndexServer extends AbstractDistributedIndexServer { } BlurIndex index; - + BlurNRTIndex writer = new BlurNRTIndex(shardContext, _mergeScheduler, dir, _gc, _searchExecutor); + + // BlurIndexNRTSimple writer = new BlurIndexNRTSimple(shardContext, + // _mergeScheduler, dir, _gc, _searchExecutor, + // _indexCloser, _refresher); + if (_clusterStatus.isReadOnly(true, _cluster, table)) { index = new BlurIndexReadOnly(writer); } else { @@ -482,7 +488,7 @@ public class DistributedIndexServer extends AbstractDistributedIndexServer { @Override public void run() { try { - final IndexSearcherClosable searcher = index.getIndexReader(); + final IndexSearcherClosable searcher = index.getIndexSearcher(); IndexReader reader = searcher.getIndexReader(); _warmup.warmBlurIndex(table, shard, reader, index.isClosed(), new ReleaseReader() { @Override http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e1a7fe6c/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndex.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndex.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndex.java index 19b26b6..8100a58 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndex.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndex.java @@ -24,8 +24,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.apache.blur.server.IndexSearcherClosable; import org.apache.blur.thrift.generated.Row; import org.apache.blur.utils.BlurUtil; -import org.apache.blur.utils.ClassNameFilter; -import org.apache.blur.utils.RamUsageEstimator; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.search.IndexSearcher; @@ -43,7 +41,7 @@ public abstract class BlurIndex { public abstract void deleteRow(boolean waitToBeVisible, boolean wal, String rowId) throws IOException; - public abstract IndexSearcherClosable getIndexReader() throws IOException; + public abstract IndexSearcherClosable getIndexSearcher() throws IOException; public abstract void close() throws IOException; @@ -60,7 +58,7 @@ public abstract class BlurIndex { public abstract List<String> getSnapshots() throws IOException; public long getRecordCount() throws IOException { - IndexSearcherClosable searcher = getIndexReader(); + IndexSearcherClosable searcher = getIndexSearcher(); try { return searcher.getIndexReader().numDocs(); } finally { @@ -71,7 +69,7 @@ public abstract class BlurIndex { } public long getRowCount() throws IOException { - IndexSearcherClosable searcher = getIndexReader(); + IndexSearcherClosable searcher = getIndexSearcher(); try { return getRowCount(searcher); } finally { @@ -87,33 +85,34 @@ public abstract class BlurIndex { } public long getIndexMemoryUsage() throws IOException { - long now = System.currentTimeMillis(); - if (_lastMemoryCheck + ONE_MINUTE > now) { - return _memoryUsage; - } - IndexSearcherClosable searcher = getIndexReader(); - try { - IndexReaderContext topReaderContext = searcher.getTopReaderContext(); - return _memoryUsage = RamUsageEstimator.sizeOf(topReaderContext, new ClassNameFilter() { - @Override - public boolean include(String className) { - if (className.startsWith("org.apache.blur.index.ExitableReader")) { - return true; - } else if (className.startsWith("org.apache.blur.")) { - // System.out.println("className [" + className + "]"); - return false; - } - return true; - } - }); - } finally { - searcher.close(); - _lastMemoryCheck = System.currentTimeMillis(); - } + return 0; +// long now = System.currentTimeMillis(); +// if (_lastMemoryCheck + ONE_MINUTE > now) { +// return _memoryUsage; +// } +// IndexSearcherClosable searcher = getIndexReader(); +// try { +// IndexReaderContext topReaderContext = searcher.getTopReaderContext(); +// return _memoryUsage = RamUsageEstimator.sizeOf(topReaderContext, new ClassNameFilter() { +// @Override +// public boolean include(String className) { +// if (className.startsWith("org.apache.blur.index.ExitableReader")) { +// return true; +// } else if (className.startsWith("org.apache.blur.")) { +// // System.out.println("className [" + className + "]"); +// return false; +// } +// return true; +// } +// }); +// } finally { +// searcher.close(); +// _lastMemoryCheck = System.currentTimeMillis(); +// } } public long getSegmentCount() throws IOException { - IndexSearcherClosable indexSearcherClosable = getIndexReader(); + IndexSearcherClosable indexSearcherClosable = getIndexSearcher(); try { IndexReader indexReader = indexSearcherClosable.getIndexReader(); IndexReaderContext context = indexReader.getContext(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e1a7fe6c/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReadOnly.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReadOnly.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReadOnly.java index 1092439..ddd82e1 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReadOnly.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReadOnly.java @@ -39,8 +39,8 @@ public class BlurIndexReadOnly extends BlurIndex { throw new IOException("Read Only"); } - public IndexSearcherClosable getIndexReader() throws IOException { - return _blurIndex.getIndexReader(); + public IndexSearcherClosable getIndexSearcher() throws IOException { + return _blurIndex.getIndexSearcher(); } public void close() throws IOException { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e1a7fe6c/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java index eb4b65e..a4653a5 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java @@ -112,7 +112,7 @@ public class BlurIndexReader extends BlurIndex { } @Override - public IndexSearcherClosable getIndexReader() throws IOException { + public IndexSearcherClosable getIndexSearcher() throws IOException { final DirectoryReader reader = _indexReaderRef.get(); reader.incRef(); return new IndexSearcherClosable(reader, null) { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e1a7fe6c/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java index f6912f2..7e77db2 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java @@ -233,7 +233,7 @@ public class BlurNRTIndex extends BlurIndex { * responsible for calling close on the searcher. */ @Override - public IndexSearcherClosable getIndexReader() throws IOException { + public IndexSearcherClosable getIndexSearcher() throws IOException { return resetRunning((IndexSearcherClosable) getNRTManager().acquire()); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e1a7fe6c/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java b/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java index 05adfb5..c799376 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java @@ -400,7 +400,7 @@ public class TransactionRecorder extends TimerTask implements Closeable { return fields; } - private Term createRowId(String id) { + public static Term createRowId(String id) { return new Term(BlurConstants.ROW_ID, id); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e1a7fe6c/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java index 5c80a9e..01762b1 100644 --- a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java +++ b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java @@ -119,12 +119,12 @@ public class BlurIndexReaderTest { @Test public void testBlurIndexWriter() throws IOException, InterruptedException, URISyntaxException { setupWriter(configuration, 1); - IndexSearcherClosable indexReader1 = reader.getIndexReader(); + IndexSearcherClosable indexReader1 = reader.getIndexSearcher(); doWrite(); assertEquals(0, indexReader1.getIndexReader().numDocs()); indexReader1.close(); reader.refresh(); - IndexSearcherClosable indexReader2 = reader.getIndexReader(); + IndexSearcherClosable indexReader2 = reader.getIndexSearcher(); assertEquals(1, indexReader2.getIndexReader().numDocs()); indexReader2.close(); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e1a7fe6c/blur-core/src/test/java/org/apache/blur/manager/writer/BlurNRTIndexTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurNRTIndexTest.java b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurNRTIndexTest.java index 9800eab..5f40fac 100644 --- a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurNRTIndexTest.java +++ b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurNRTIndexTest.java @@ -129,7 +129,7 @@ public class BlurNRTIndexTest { int total = 0; for (int i = 0; i < TEST_NUMBER_WAIT_VISIBLE; i++) { writer.replaceRow(true, true, genRow()); - IndexSearcherClosable searcher = writer.getIndexReader(); + IndexSearcherClosable searcher = writer.getIndexSearcher(); IndexReader reader = searcher.getIndexReader(); assertEquals(i + 1, reader.numDocs()); searcher.close(); @@ -139,7 +139,7 @@ public class BlurNRTIndexTest { double seconds = (e - s) / 1000000000.0; double rate = total / seconds; System.out.println("Rate " + rate); - IndexSearcherClosable searcher = writer.getIndexReader(); + IndexSearcherClosable searcher = writer.getIndexSearcher(); IndexReader reader = searcher.getIndexReader(); assertEquals(TEST_NUMBER_WAIT_VISIBLE, reader.numDocs()); searcher.close(); @@ -148,7 +148,7 @@ public class BlurNRTIndexTest { @Test public void testBlurIndexWriterFaster() throws IOException, InterruptedException { setupWriter(configuration, 100, false); - IndexSearcherClosable searcher1 = writer.getIndexReader(); + IndexSearcherClosable searcher1 = writer.getIndexSearcher(); IndexReader reader1 = searcher1.getIndexReader(); assertEquals(0, reader1.numDocs()); searcher1.close(); @@ -170,7 +170,7 @@ public class BlurNRTIndexTest { // refresh once every 25 ms // Thread.sleep(1000); writer.refresh(); - IndexSearcherClosable searcher2 = writer.getIndexReader(); + IndexSearcherClosable searcher2 = writer.getIndexSearcher(); IndexReader reader2 = searcher2.getIndexReader(); assertEquals(TEST_NUMBER, reader2.numDocs()); searcher2.close(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e1a7fe6c/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java index f29ef14..6b07d04 100644 --- a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java +++ b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java @@ -364,6 +364,8 @@ public class BlurClusterTest { + getMaxHeapSize() + "]"); bufferToPutGcWatcherOverLimitList.add(new byte[sizeToAllocate]); } else { + System.gc(); + System.gc(); System.out.println("Already allocated enough Heap [" + getHeapSize() + "] Max [" + getMaxHeapSize() + "]"); } }
