Fixing more unit tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/b73a6ace Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/b73a6ace Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/b73a6ace Branch: refs/heads/master Commit: b73a6ace343e3e151570955dc64f1b8c39ed8531 Parents: ed04d5b Author: Aaron McCurry <[email protected]> Authored: Thu Jun 6 22:45:25 2013 -0400 Committer: Aaron McCurry <[email protected]> Committed: Thu Jun 6 22:45:25 2013 -0400 ---------------------------------------------------------------------- .../blur/manager/writer/BlurIndexReader.java | 36 +++++----- .../blur/manager/writer/BlurNRTIndex.java | 3 +- .../blur/server/IndexSearcherClosable.java | 36 +++++----- .../blur/server/IndexSearcherClosableNRT.java | 49 ++++++++++++++ .../blur/manager/indexserver/SafeModeTest.java | 5 +- .../manager/writer/BlurIndexReaderTest.java | 71 ++++++++++---------- 6 files changed, 130 insertions(+), 70 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b73a6ace/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java index 431a829..f31fa38 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java @@ -31,10 +31,7 @@ import org.apache.blur.server.ShardContext; import org.apache.blur.server.TableContext; import org.apache.blur.thrift.generated.Row; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.TieredMergePolicy; -import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.store.Directory; public class BlurIndexReader extends BlurIndex { @@ -57,16 +54,14 @@ public class BlurIndexReader extends BlurIndex { _shardContext = shardContext; _refresher = refresher; _closer = closer; - IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, _tableContext.getAnalyzer()); - conf.setWriteLockTimeout(TimeUnit.MINUTES.toMillis(5)); - conf.setIndexDeletionPolicy(_tableContext.getIndexDeletionPolicy()); - conf.setSimilarity(_tableContext.getSimilarity()); - TieredMergePolicy mergePolicy = (TieredMergePolicy) conf.getMergePolicy(); - mergePolicy.setUseCompoundFile(false); _open.set(true); if (!DirectoryReader.indexExists(directory)) { + LOG.info("Creating an empty index"); + // if the directory is empty then create an empty index. + IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, _tableContext.getAnalyzer()); + conf.setWriteLockTimeout(TimeUnit.MINUTES.toMillis(5)); new IndexWriter(directory, conf).close(); } _indexReaderRef.set(DirectoryReader.open(directory)); @@ -107,12 +102,25 @@ public class BlurIndexReader extends BlurIndex { @Override public void optimize(int numberOfSegmentsPerShard) throws IOException { - // Do nothing + throw new RuntimeException("Read-only shard"); } @Override public IndexSearcherClosable getIndexReader() throws IOException { - throw new RuntimeException("not implemented"); + final DirectoryReader reader = _indexReaderRef.get(); + reader.incRef(); + return new IndexSearcherClosable(reader, null) { + + @Override + public Directory getDirectory() { + return _directory; + } + + @Override + public void close() throws IOException { + reader.decRef(); + } + }; } @Override @@ -120,10 +128,4 @@ public class BlurIndexReader extends BlurIndex { return _isClosed; } - public IndexSearcher getSearcher() { - IndexReader indexReader = _indexReaderRef.get(); - indexReader.incRef(); - return new IndexSearcher(indexReader); - } - } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b73a6ace/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java index 9adbc46..43310fb 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java @@ -35,6 +35,7 @@ import org.apache.blur.lucene.store.refcounter.DirectoryReferenceCounter; import org.apache.blur.lucene.store.refcounter.DirectoryReferenceFileGC; import org.apache.blur.lucene.store.refcounter.IndexInputCloser; import org.apache.blur.server.IndexSearcherClosable; +import org.apache.blur.server.IndexSearcherClosableNRT; import org.apache.blur.server.ShardContext; import org.apache.blur.server.TableContext; import org.apache.blur.thrift.generated.Record; @@ -97,7 +98,7 @@ public class BlurNRTIndex extends BlurIndex { _searcherFactory = new SearcherFactory() { @Override public IndexSearcher newSearcher(IndexReader reader) throws IOException { - return new IndexSearcherClosable(reader, searchExecutor, _nrtManagerRef, _directory); + return new IndexSearcherClosableNRT(reader, searchExecutor, _nrtManagerRef, _directory); } }; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b73a6ace/blur-core/src/main/java/org/apache/blur/server/IndexSearcherClosable.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/server/IndexSearcherClosable.java b/blur-core/src/main/java/org/apache/blur/server/IndexSearcherClosable.java index 64574d8..331a9ff 100644 --- a/blur-core/src/main/java/org/apache/blur/server/IndexSearcherClosable.java +++ b/blur-core/src/main/java/org/apache/blur/server/IndexSearcherClosable.java @@ -1,34 +1,38 @@ package org.apache.blur.server; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ import java.io.Closeable; import java.io.IOException; import java.util.concurrent.ExecutorService; -import java.util.concurrent.atomic.AtomicReference; import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.NRTManager; import org.apache.lucene.store.Directory; -public class IndexSearcherClosable extends IndexSearcher implements Closeable { +public abstract class IndexSearcherClosable extends IndexSearcher implements Closeable { - private final AtomicReference<NRTManager> _nrtManagerRef; - private final Directory _directory; - - public IndexSearcherClosable(IndexReader r, ExecutorService executor, AtomicReference<NRTManager> nrtManagerRef, - Directory directory) { + public IndexSearcherClosable(IndexReader r, ExecutorService executor) { super(r, executor); - _nrtManagerRef = nrtManagerRef; - _directory = directory; } - public Directory getDirectory() { - return _directory; - } + public abstract Directory getDirectory(); @Override - public void close() throws IOException { - _nrtManagerRef.get().release(this); - } + public abstract void close() throws IOException; } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b73a6ace/blur-core/src/main/java/org/apache/blur/server/IndexSearcherClosableNRT.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/server/IndexSearcherClosableNRT.java b/blur-core/src/main/java/org/apache/blur/server/IndexSearcherClosableNRT.java new file mode 100644 index 0000000..ddd28ab --- /dev/null +++ b/blur-core/src/main/java/org/apache/blur/server/IndexSearcherClosableNRT.java @@ -0,0 +1,49 @@ +package org.apache.blur.server; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +import java.io.IOException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicReference; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.NRTManager; +import org.apache.lucene.store.Directory; + +public class IndexSearcherClosableNRT extends IndexSearcherClosable { + + private final AtomicReference<NRTManager> _nrtManagerRef; + private final Directory _directory; + + public IndexSearcherClosableNRT(IndexReader r, ExecutorService executor, AtomicReference<NRTManager> nrtManagerRef, + Directory directory) { + super(r, executor); + _nrtManagerRef = nrtManagerRef; + _directory = directory; + } + + @Override + public Directory getDirectory() { + return _directory; + } + + @Override + public void close() throws IOException { + _nrtManagerRef.get().release(this); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b73a6ace/blur-core/src/test/java/org/apache/blur/manager/indexserver/SafeModeTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/manager/indexserver/SafeModeTest.java b/blur-core/src/test/java/org/apache/blur/manager/indexserver/SafeModeTest.java index c568cd1..da06232 100644 --- a/blur-core/src/test/java/org/apache/blur/manager/indexserver/SafeModeTest.java +++ b/blur-core/src/test/java/org/apache/blur/manager/indexserver/SafeModeTest.java @@ -77,7 +77,8 @@ public class SafeModeTest { } for (AtomicReference<Throwable> t : errors) { - assertNull(t.get()); + Throwable throwable = t.get(); + assertNull(throwable == null ? null : throwable.getMessage(), throwable); } long oldest = -1; @@ -91,7 +92,7 @@ public class SafeModeTest { newest = l; } } - assertTrue((newest - oldest) < TimeUnit.SECONDS.toMillis(1)); + assertTrue("newest [" + newest + "] oldest [" + oldest + "]", (newest - oldest) < TimeUnit.SECONDS.toMillis(5)); } @Test http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b73a6ace/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java index f225cfc..76886e4 100644 --- a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java +++ b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java @@ -21,32 +21,34 @@ import static org.junit.Assert.assertEquals; import java.io.File; import java.io.IOException; -import java.util.Random; +import java.util.UUID; import java.util.concurrent.ExecutorService; import org.apache.blur.concurrent.Executors; +import org.apache.blur.index.IndexWriter; import org.apache.blur.lucene.store.refcounter.DirectoryReferenceFileGC; import org.apache.blur.lucene.store.refcounter.IndexInputCloser; +import org.apache.blur.server.IndexSearcherClosable; import org.apache.blur.server.ShardContext; import org.apache.blur.server.TableContext; import org.apache.blur.thrift.generated.AnalyzerDefinition; -import org.apache.blur.thrift.generated.Column; -import org.apache.blur.thrift.generated.Record; -import org.apache.blur.thrift.generated.Row; import org.apache.blur.thrift.generated.TableDescriptor; import org.apache.hadoop.conf.Configuration; -import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.analysis.core.KeywordAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.LockObtainFailedException; +import org.apache.lucene.util.Version; import org.junit.After; import org.junit.Before; import org.junit.Test; public class BlurIndexReaderTest { - private static final File TMPDIR = new File("./target/tmp"); + private static final File TMPDIR = new File("./target/tmp/BlurIndexReaderTest"); - private BlurNRTIndex writer; - private Random random = new Random(); private ExecutorService service; private File base; private Configuration configuration; @@ -56,6 +58,10 @@ public class BlurIndexReaderTest { private SharedMergeScheduler mergeScheduler; private BlurIndexReader reader; + private BlurIndexRefresher refresher; + private BlurIndexCloser indexCloser; + private FSDirectory directory; + @Before public void setup() throws IOException { TableContext.clear(); @@ -71,25 +77,23 @@ public class BlurIndexReaderTest { configuration = new Configuration(); service = Executors.newThreadPool("test", 1); - + } private void setupWriter(Configuration configuration, long refresh) throws IOException { TableDescriptor tableDescriptor = new TableDescriptor(); tableDescriptor.setName("test-table"); - tableDescriptor.setTableUri(new File(base, "table-store").toURI().toString()); + tableDescriptor.setTableUri(new File(base, "table-store-" + UUID.randomUUID().toString()).toURI().toString()); tableDescriptor.setAnalyzerDefinition(new AnalyzerDefinition()); tableDescriptor.putToTableProperties("blur.shard.time.between.refreshs", Long.toString(refresh)); tableDescriptor.putToTableProperties("blur.shard.time.between.commits", Long.toString(1000)); - + TableContext tableContext = TableContext.create(tableDescriptor); - FSDirectory directory = FSDirectory.open(new File(base, "index")); + directory = FSDirectory.open(new File(tableDescriptor.getTableUri())); ShardContext shardContext = ShardContext.create(tableContext, "test-shard"); - - writer = new BlurNRTIndex(shardContext, mergeScheduler, closer, directory, gc, service); - BlurIndexRefresher refresher = new BlurIndexRefresher(); - BlurIndexCloser indexCloser = new BlurIndexCloser(); + refresher = new BlurIndexRefresher(); + indexCloser = new BlurIndexCloser(); refresher.init(); indexCloser.init(); reader = new BlurIndexReader(shardContext, directory, refresher, indexCloser); @@ -97,7 +101,6 @@ public class BlurIndexReaderTest { @After public void tearDown() throws IOException { - writer.close(); mergeScheduler.close(); closer.close(); gc.close(); @@ -120,25 +123,25 @@ public class BlurIndexReaderTest { @Test public void testBlurIndexWriter() throws IOException, InterruptedException { setupWriter(configuration, 1); - IndexSearcher searcher = reader.getSearcher(); - writer.replaceRow(true, true, genRow()); - Thread.sleep(1500); - assertEquals(0,searcher.getIndexReader().numDocs()); + IndexSearcherClosable indexReader1 = reader.getIndexReader(); + doWrite(); + assertEquals(0, indexReader1.getIndexReader().numDocs()); + indexReader1.close(); reader.refresh(); - assertEquals(1,reader.getSearcher().getIndexReader().numDocs()); + IndexSearcherClosable indexReader2 = reader.getIndexReader(); + assertEquals(1, indexReader2.getIndexReader().numDocs()); + indexReader2.close(); } - - private Row genRow() { - Row row = new Row(); - row.setId(Long.toString(random.nextLong())); - Record record = new Record(); - record.setFamily("testing"); - record.setRecordId(Long.toString(random.nextLong())); - for (int i = 0; i < 10; i++) { - record.addToColumns(new Column("col" + i, Long.toString(random.nextLong()))); - } - row.addToRecords(record); - return row; + + private void doWrite() throws CorruptIndexException, LockObtainFailedException, IOException { + IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer()); + IndexWriter writer = new IndexWriter(directory, conf); + writer.addDocument(getDoc()); + writer.close(); + } + + private Document getDoc() { + return new Document(); } }
