Adding record level security as a builtin feature of Blur. More testing needs to be created to consider this task complete, however the current implementation is complete.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/0b065b16 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/0b065b16 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/0b065b16 Branch: refs/heads/master Commit: 0b065b1619d3491c4aa5e9a6f88321b41fef164d Parents: a298111 Author: Aaron McCurry <[email protected]> Authored: Fri Jan 16 09:34:20 2015 -0500 Committer: Aaron McCurry <[email protected]> Committed: Fri Jan 16 09:34:20 2015 -0500 ---------------------------------------------------------------------- .../apache/blur/command/CoreTestContext.java | 6 +- .../blur/command/TestContextArgDecorator.java | 4 +- .../org/apache/blur/command/IndexContext.java | 4 +- .../blur/command/ShardCommandManager.java | 13 +- .../org/apache/blur/manager/IndexManager.java | 45 +- .../results/BlurResultIterableSearcher.java | 6 +- .../apache/blur/manager/writer/BlurIndex.java | 13 +- .../blur/manager/writer/BlurIndexReadOnly.java | 4 +- .../manager/writer/BlurIndexSimpleWriter.java | 183 +++- .../apache/blur/manager/writer/IndexAction.java | 6 +- .../blur/manager/writer/IndexImporter.java | 6 +- .../blur/manager/writer/MutatableAction.java | 16 +- .../manager/writer/MutationQueueProcessor.java | 6 +- .../blur/server/IndexSearcherClosable.java | 76 -- .../blur/server/IndexSearcherCloseableBase.java | 75 ++ .../IndexSearcherCloseableSecureBase.java | 76 ++ .../apache/blur/server/ShardServerContext.java | 21 +- .../org/apache/blur/server/TableContext.java | 22 + .../org/apache/blur/utils/GCWatcherJdk7.java | 47 +- .../test/java/org/apache/blur/MiniCluster.java | 4 +- .../blur/command/ShardCommandManagerTest.java | 19 +- .../writer/BlurIndexSimpleWriterTest.java | 19 +- .../blur/manager/writer/IndexImporterTest.java | 19 +- .../manager/writer/MutatableActionTest.java | 18 +- .../org/apache/blur/thrift/BlurClusterTest.java | 926 ------------------ .../apache/blur/thrift/BlurClusterTestBase.java | 965 +++++++++++++++++++ .../blur/thrift/BlurClusterTestNoSecurity.java | 54 ++ .../blur/thrift/BlurClusterTestSecurity.java | 83 ++ blur-query/pom.xml | 5 - .../lucene/search/IndexSearcherCloseable.java | 40 + .../search/IndexSearcherCloseableUtil.java | 65 ++ .../blur/lucene/search/IterablePaging.java | 5 +- .../lucene/search/TestingPagingCollector.java | 6 +- blur-store/pom.xml | 37 +- .../org/apache/blur/index/AtomicReaderUtil.java | 7 + .../org/apache/blur/utils/BlurConstants.java | 5 + .../lucene-document-security-0.1.0-tests.jar | Bin 18023 -> 0 bytes .../0.1.0/lucene-document-security-0.1.0.jar | Bin 80656 -> 0 bytes .../0.1.0/lucene-document-security-0.1.0.pom | 117 --- .../lucene-document-security-0.1.1-tests.jar | Bin 0 -> 17894 bytes .../0.1.1/lucene-document-security-0.1.1.jar | Bin 0 -> 79268 bytes .../0.1.1/lucene-document-security-0.1.1.pom | 117 +++ 42 files changed, 1831 insertions(+), 1309 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-command/src/test/java/org/apache/blur/command/CoreTestContext.java ---------------------------------------------------------------------- diff --git a/blur-command/src/test/java/org/apache/blur/command/CoreTestContext.java b/blur-command/src/test/java/org/apache/blur/command/CoreTestContext.java index 838b03d..f12f815 100644 --- a/blur-command/src/test/java/org/apache/blur/command/CoreTestContext.java +++ b/blur-command/src/test/java/org/apache/blur/command/CoreTestContext.java @@ -20,6 +20,8 @@ package org.apache.blur.command; import java.io.IOException; import org.apache.blur.BlurConfiguration; +import org.apache.blur.lucene.search.IndexSearcherCloseable; +import org.apache.blur.lucene.search.IndexSearcherCloseableUtil; import org.apache.blur.server.TableContext; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.Document; @@ -88,8 +90,8 @@ public class CoreTestContext extends IndexContext { } @Override - public IndexSearcher getIndexSearcher() { - return new IndexSearcher(getIndexReader()); + public IndexSearcherCloseable getIndexSearcher() { + return IndexSearcherCloseableUtil.wrap(new IndexSearcher(getIndexReader())); } @Override http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-command/src/test/java/org/apache/blur/command/TestContextArgDecorator.java ---------------------------------------------------------------------- diff --git a/blur-command/src/test/java/org/apache/blur/command/TestContextArgDecorator.java b/blur-command/src/test/java/org/apache/blur/command/TestContextArgDecorator.java index 295d253..73f03a2 100644 --- a/blur-command/src/test/java/org/apache/blur/command/TestContextArgDecorator.java +++ b/blur-command/src/test/java/org/apache/blur/command/TestContextArgDecorator.java @@ -3,9 +3,9 @@ package org.apache.blur.command; import java.io.IOException; import org.apache.blur.BlurConfiguration; +import org.apache.blur.lucene.search.IndexSearcherCloseable; import org.apache.blur.server.TableContext; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.search.IndexSearcher; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -43,7 +43,7 @@ public class TestContextArgDecorator extends IndexContext { return heavyContext.getIndexReader(); } - public IndexSearcher getIndexSearcher() { + public IndexSearcherCloseable getIndexSearcher() { return heavyContext.getIndexSearcher(); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-core/src/main/java/org/apache/blur/command/IndexContext.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/command/IndexContext.java b/blur-core/src/main/java/org/apache/blur/command/IndexContext.java index 97f0e09..ad3074e 100644 --- a/blur-core/src/main/java/org/apache/blur/command/IndexContext.java +++ b/blur-core/src/main/java/org/apache/blur/command/IndexContext.java @@ -3,9 +3,9 @@ package org.apache.blur.command; import java.io.IOException; import org.apache.blur.BlurConfiguration; +import org.apache.blur.lucene.search.IndexSearcherCloseable; import org.apache.blur.server.TableContext; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.search.IndexSearcher; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -32,7 +32,7 @@ public abstract class IndexContext extends BaseContext { public abstract IndexReader getIndexReader(); - public abstract IndexSearcher getIndexSearcher(); + public abstract IndexSearcherCloseable getIndexSearcher(); public abstract BlurConfiguration getBlurConfiguration() throws IOException; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-core/src/main/java/org/apache/blur/command/ShardCommandManager.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/command/ShardCommandManager.java b/blur-core/src/main/java/org/apache/blur/command/ShardCommandManager.java index cbf218f..d518523 100644 --- a/blur-core/src/main/java/org/apache/blur/command/ShardCommandManager.java +++ b/blur-core/src/main/java/org/apache/blur/command/ShardCommandManager.java @@ -28,15 +28,14 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import org.apache.blur.BlurConfiguration; +import org.apache.blur.lucene.search.IndexSearcherCloseable; import org.apache.blur.manager.IndexServer; import org.apache.blur.manager.writer.BlurIndex; -import org.apache.blur.server.IndexSearcherClosable; import org.apache.blur.server.ShardServerContext; import org.apache.blur.server.TableContext; import org.apache.blur.server.TableContextFactory; import org.apache.hadoop.conf.Configuration; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.search.IndexSearcher; public class ShardCommandManager extends BaseCommandManager { @@ -195,7 +194,7 @@ public class ShardCommandManager extends BaseCommandManager { @Override public Object call() throws Exception { String shardId = shard.getShard(); - IndexSearcherClosable searcher = shardServerContext.getIndexSearcherClosable(table, shardId); + IndexSearcherCloseable searcher = shardServerContext.getIndexSearcherClosable(table, shardId); if (searcher == null) { searcher = blurIndex.getIndexSearcher(); shardServerContext.setIndexSearcherClosable(table, shardId, searcher); @@ -213,7 +212,7 @@ public class ShardCommandManager extends BaseCommandManager { public Object call() throws Exception { String shardId = shard.getShard(); - IndexSearcherClosable searcher = shardServerContext.getIndexSearcherClosable(table, shardId); + IndexSearcherCloseable searcher = shardServerContext.getIndexSearcherClosable(table, shardId); if (searcher == null) { searcher = blurIndex.getIndexSearcher(); shardServerContext.setIndexSearcherClosable(table, shardId, searcher); @@ -226,11 +225,11 @@ public class ShardCommandManager extends BaseCommandManager { static class ShardIndexContext extends IndexContext { private final Shard _shard; - private final IndexSearcher _searcher; + private final IndexSearcherCloseable _searcher; private final TableContextFactory _tableContextFactory; private final String _table; - public ShardIndexContext(TableContextFactory tableContextFactory, String table, Shard shard, IndexSearcher searcher) { + public ShardIndexContext(TableContextFactory tableContextFactory, String table, Shard shard, IndexSearcherCloseable searcher) { _tableContextFactory = tableContextFactory; _table = table; _shard = shard; @@ -243,7 +242,7 @@ public class ShardCommandManager extends BaseCommandManager { } @Override - public IndexSearcher getIndexSearcher() { + public IndexSearcherCloseable getIndexSearcher() { return _searcher; } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java index 2c6c249..cea6f68 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java +++ b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java @@ -46,6 +46,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLongArray; +import lucene.security.index.SecureDirectoryReader; + import org.apache.blur.analysis.FieldManager; import org.apache.blur.concurrent.Executors; import org.apache.blur.index.AtomicReaderUtil; @@ -56,6 +58,7 @@ import org.apache.blur.log.LogFactory; import org.apache.blur.lucene.search.DeepPagingCache; import org.apache.blur.lucene.search.FacetExecutor; import org.apache.blur.lucene.search.FacetQuery; +import org.apache.blur.lucene.search.IndexSearcherCloseable; import org.apache.blur.lucene.search.StopExecutionCollector.StopExecutionCollectorException; import org.apache.blur.manager.clusterstatus.ClusterStatus; import org.apache.blur.manager.results.BlurResultIterable; @@ -65,7 +68,6 @@ import org.apache.blur.manager.status.QueryStatus; import org.apache.blur.manager.status.QueryStatusManager; import org.apache.blur.manager.writer.BlurIndex; import org.apache.blur.manager.writer.MutatableAction; -import org.apache.blur.server.IndexSearcherClosable; import org.apache.blur.server.ShardContext; import org.apache.blur.server.ShardServerContext; import org.apache.blur.server.TableContext; @@ -105,6 +107,7 @@ import org.apache.lucene.index.AtomicReader; import org.apache.lucene.index.BaseCompositeReader; import org.apache.lucene.index.BaseCompositeReaderUtil; import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.MultiFields; @@ -270,7 +273,7 @@ public class IndexManager { BlurIndex index = null; String shard = null; Tracer trace = Trace.trace("manager fetch", Trace.param("table", table)); - IndexSearcherClosable searcher = null; + IndexSearcherCloseable searcher = null; try { if (selector.getLocationId() == null) { // Not looking up by location id so we should resetSearchers. @@ -371,7 +374,7 @@ public class IndexManager { getScoreType(query.scoreType), context); } - public static void populateSelector(IndexSearcherClosable searcher, String shardName, String table, Selector selector) + public static void populateSelector(IndexSearcherCloseable searcher, String shardName, String table, Selector selector) throws IOException { Tracer trace = Trace.trace("populate selector"); String rowId = selector.rowId; @@ -944,7 +947,7 @@ public class IndexManager { @Override public Long call(Entry<String, BlurIndex> input) throws Exception { BlurIndex index = input.getValue(); - IndexSearcherClosable searcher = index.getIndexSearcher(); + IndexSearcherCloseable searcher = index.getIndexSearcher(); try { return recordFrequency(searcher.getIndexReader(), columnFamily, columnName, value); } finally { @@ -980,7 +983,7 @@ public class IndexManager { @Override public List<String> call(Entry<String, BlurIndex> input) throws Exception { BlurIndex index = input.getValue(); - IndexSearcherClosable searcher = index.getIndexSearcher(); + IndexSearcherCloseable searcher = index.getIndexSearcher(); try { return terms(searcher.getIndexReader(), columnFamily, columnName, startWith, size); } finally { @@ -1214,15 +1217,12 @@ public class IndexManager { String shard = entry.getKey(); _status.attachThread(shard); BlurIndex index = entry.getValue(); - IndexSearcherClosable searcher = index.getIndexSearcher(); + IndexSearcherCloseable searcher = index.getIndexSearcher(); Tracer trace2 = null; try { IndexReader indexReader = searcher.getIndexReader(); - if (indexReader instanceof ExitableReader) { - ExitableReader er = (ExitableReader) indexReader; - er.setRunning(_running); - } else { - throw new IOException("IndexReader is not ExitableReader"); + if (!resetExitableReader(indexReader, _running)) { + throw new IOException("Cannot find ExitableReader in stack."); } if (_shardServerContext != null) { _shardServerContext.setIndexSearcherClosable(_table, shard, searcher); @@ -1264,6 +1264,21 @@ public class IndexManager { _status.deattachThread(shard); } } + + } + + private static boolean resetExitableReader(IndexReader indexReader, AtomicBoolean running) { + if (indexReader instanceof ExitableReader) { + ExitableReader exitableReader = (ExitableReader) indexReader; + exitableReader.setRunning(running); + return true; + } + if (indexReader instanceof SecureDirectoryReader) { + SecureDirectoryReader secureDirectoryReader = (SecureDirectoryReader) indexReader; + DirectoryReader original = secureDirectoryReader.getOriginal(); + return resetExitableReader(original, running); + } + return false; } public void optimize(String table, int numberOfSegmentsPerShard) throws BException { @@ -1307,14 +1322,16 @@ public class IndexManager { blurIndex.addBulkMutate(bulkId, mutation); } - public void bulkMutateFinish(String table, String bulkId, boolean apply, boolean blockUntilComplete) throws BlurException, IOException { + public void bulkMutateFinish(String table, String bulkId, boolean apply, boolean blockUntilComplete) + throws BlurException, IOException { Map<String, BlurIndex> indexes = _indexServer.getIndexes(table); for (BlurIndex index : indexes.values()) { - index.finishBulkMutate(bulkId, apply,blockUntilComplete); + index.finishBulkMutate(bulkId, apply, blockUntilComplete); } } - public void bulkMutateAddMultiple(String table, String bulkId, List<RowMutation> rowMutations) throws BlurException, IOException { + public void bulkMutateAddMultiple(String table, String bulkId, List<RowMutation> rowMutations) throws BlurException, + IOException { for (RowMutation mutation : rowMutations) { bulkMutateAdd(table, bulkId, mutation); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-core/src/main/java/org/apache/blur/manager/results/BlurResultIterableSearcher.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/results/BlurResultIterableSearcher.java b/blur-core/src/main/java/org/apache/blur/manager/results/BlurResultIterableSearcher.java index 000a9cf..ac1517b 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/results/BlurResultIterableSearcher.java +++ b/blur-core/src/main/java/org/apache/blur/manager/results/BlurResultIterableSearcher.java @@ -22,10 +22,10 @@ import java.util.TreeMap; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.blur.lucene.search.DeepPagingCache; +import org.apache.blur.lucene.search.IndexSearcherCloseable; import org.apache.blur.lucene.search.IterablePaging; import org.apache.blur.lucene.search.IterablePaging.ProgressRef; import org.apache.blur.lucene.search.IterablePaging.TotalHitsRef; -import org.apache.blur.server.IndexSearcherClosable; import org.apache.blur.server.TableContext; import org.apache.blur.thrift.generated.BlurException; import org.apache.blur.thrift.generated.BlurResult; @@ -54,11 +54,11 @@ public class BlurResultIterableSearcher implements BlurResultIterable { private final IterablePaging _iterablePaging; private final Sort _sort; - private IndexSearcherClosable _searcher; + private IndexSearcherCloseable _searcher; private long _skipTo; public BlurResultIterableSearcher(AtomicBoolean running, Query query, String table, String shard, - IndexSearcherClosable searcher, Selector selector, boolean closeSearcher, boolean runSlow, int fetchCount, + IndexSearcherCloseable searcher, Selector selector, boolean closeSearcher, boolean runSlow, int fetchCount, int maxHeapPerRowFetch, TableContext context, Sort sort, DeepPagingCache deepPagingCache) throws BlurException { _sort = sort; _running = running; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndex.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndex.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndex.java index eec3f65..f098ad7 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndex.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndex.java @@ -23,13 +23,12 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import org.apache.blur.server.IndexSearcherClosable; +import org.apache.blur.lucene.search.IndexSearcherCloseable; import org.apache.blur.server.ShardContext; import org.apache.blur.thrift.generated.RowMutation; import org.apache.blur.utils.BlurUtil; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReaderContext; -import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; @@ -46,7 +45,7 @@ public abstract class BlurIndex { _shardContext = shardContext; } - public abstract IndexSearcherClosable getIndexSearcher() throws IOException; + public abstract IndexSearcherCloseable getIndexSearcher() throws IOException; public abstract void close() throws IOException; @@ -63,7 +62,7 @@ public abstract class BlurIndex { public abstract List<String> getSnapshots() throws IOException; public long getRecordCount() throws IOException { - IndexSearcherClosable searcher = getIndexSearcher(); + IndexSearcherCloseable searcher = getIndexSearcher(); try { return searcher.getIndexReader().numDocs(); } finally { @@ -74,7 +73,7 @@ public abstract class BlurIndex { } public long getRowCount() throws IOException { - IndexSearcherClosable searcher = getIndexSearcher(); + IndexSearcherCloseable searcher = getIndexSearcher(); try { return getRowCount(searcher); } finally { @@ -84,7 +83,7 @@ public abstract class BlurIndex { } } - protected long getRowCount(IndexSearcher searcher) throws IOException { + protected long getRowCount(IndexSearcherCloseable searcher) throws IOException { TopDocs topDocs = searcher.search(new TermQuery(BlurUtil.PRIME_DOC_TERM), 1); return topDocs.totalHits; } @@ -118,7 +117,7 @@ public abstract class BlurIndex { } public long getSegmentCount() throws IOException { - IndexSearcherClosable indexSearcherClosable = getIndexSearcher(); + IndexSearcherCloseable indexSearcherClosable = getIndexSearcher(); try { IndexReader indexReader = indexSearcherClosable.getIndexReader(); IndexReaderContext context = indexReader.getContext(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReadOnly.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReadOnly.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReadOnly.java index a33dc62..6025068 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReadOnly.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReadOnly.java @@ -20,7 +20,7 @@ import java.io.IOException; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; -import org.apache.blur.server.IndexSearcherClosable; +import org.apache.blur.lucene.search.IndexSearcherCloseable; import org.apache.blur.thrift.generated.RowMutation; public class BlurIndexReadOnly extends BlurIndex { @@ -32,7 +32,7 @@ public class BlurIndexReadOnly extends BlurIndex { _blurIndex = blurIndex; } - public IndexSearcherClosable getIndexSearcher() throws IOException { + public IndexSearcherCloseable getIndexSearcher() throws IOException { return _blurIndex.getIndexSearcher(); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexSimpleWriter.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexSimpleWriter.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexSimpleWriter.java index d1483aa..afb87a6 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexSimpleWriter.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexSimpleWriter.java @@ -1,5 +1,5 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more +s * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 @@ -17,12 +17,18 @@ package org.apache.blur.manager.writer; import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION; +import static org.apache.blur.utils.BlurConstants.ACL_DISCOVER; +import static org.apache.blur.utils.BlurConstants.ACL_READ; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_QUEUE_MAX_INMEMORY_LENGTH; import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.Timer; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; @@ -37,6 +43,8 @@ import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; +import lucene.security.index.AccessControlFactory; + import org.apache.blur.BlurConfiguration; import org.apache.blur.analysis.FieldManager; import org.apache.blur.index.ExitableReader; @@ -44,13 +52,20 @@ import org.apache.blur.index.IndexDeletionPolicyReader; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; import org.apache.blur.lucene.codec.Blur024Codec; -import org.apache.blur.server.IndexSearcherClosable; +import org.apache.blur.lucene.search.IndexSearcherCloseable; +import org.apache.blur.server.IndexSearcherCloseableBase; +import org.apache.blur.server.IndexSearcherCloseableSecureBase; import org.apache.blur.server.ShardContext; import org.apache.blur.server.TableContext; import org.apache.blur.thrift.generated.BlurException; import org.apache.blur.thrift.generated.RowMutation; +import org.apache.blur.thrift.generated.TableDescriptor; import org.apache.blur.trace.Trace; import org.apache.blur.trace.Tracer; +import org.apache.blur.user.User; +import org.apache.blur.user.UserContext; +import org.apache.blur.utils.BlurConstants; +import org.apache.blur.utils.BlurUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -64,19 +79,25 @@ import org.apache.hadoop.io.SequenceFile.Writer; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.io.compress.DefaultCodec; -import org.apache.hadoop.io.compress.SnappyCodec; import org.apache.hadoop.util.Progressable; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.BlurIndexWriter; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.TieredMergePolicy; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; +import com.google.common.base.Splitter; + public class BlurIndexSimpleWriter extends BlurIndex { + private static final String TRUE = "true"; + private static final Log LOG = LogFactory.getLog(BlurIndexSimpleWriter.class); private final AtomicBoolean _isClosed = new AtomicBoolean(); @@ -106,10 +127,15 @@ public class BlurIndexSimpleWriter extends BlurIndex { private final MutationQueueProcessor _mutationQueueProcessor; private final Timer _indexImporterTimer; private final Map<String, BulkEntry> _bulkWriters; + private final boolean _security; + private final AccessControlFactory _accessControlFactory; + private final Set<String> _discoverableFields; + private final Splitter _commaSplitter; public BlurIndexSimpleWriter(ShardContext shardContext, Directory directory, SharedMergeScheduler mergeScheduler, final ExecutorService searchExecutor, BlurIndexCloser indexCloser, Timer indexImporterTimer) throws IOException { super(shardContext, directory, mergeScheduler, searchExecutor, indexCloser, indexImporterTimer); + _commaSplitter = Splitter.on(','); _bulkWriters = new ConcurrentHashMap<String, BlurIndexSimpleWriter.BulkEntry>(); _indexImporterTimer = indexImporterTimer; _searchThreadPool = searchExecutor; @@ -117,6 +143,22 @@ public class BlurIndexSimpleWriter extends BlurIndex { _tableContext = _shardContext.getTableContext(); _context = _tableContext.getTable() + "/" + shardContext.getShard(); _fieldManager = _tableContext.getFieldManager(); + _discoverableFields = _tableContext.getDiscoverableFields(); + _accessControlFactory = _tableContext.getAccessControlFactory(); + TableDescriptor descriptor = _tableContext.getDescriptor(); + Map<String, String> tableProperties = descriptor.getTableProperties(); + if (tableProperties != null) { + String value = tableProperties.get(BlurConstants.BLUR_RECORD_SECURITY); + if (value != null && value.equals(TRUE)) { + LOG.info("Record Level Security has been enabled for table [{0}] shard [{1}]", _tableContext.getTable(), + _shardContext.getShard()); + _security = true; + } else { + _security = false; + } + } else { + _security = false; + } Analyzer analyzer = _fieldManager.getAnalyzerForIndex(); _conf = new IndexWriterConfig(LUCENE_VERSION, analyzer); _conf.setWriteLockTimeout(TimeUnit.MINUTES.toMillis(5)); @@ -193,7 +235,11 @@ public class BlurIndexSimpleWriter extends BlurIndex { } @Override - public IndexSearcherClosable getIndexSearcher() throws IOException { + public IndexSearcherCloseable getIndexSearcher() throws IOException { + return getIndexSearcher(_security); + } + + public IndexSearcherCloseable getIndexSearcher(boolean security) throws IOException { final IndexReader indexReader; _indexRefreshReadLock.lock(); try { @@ -205,8 +251,65 @@ public class BlurIndexSimpleWriter extends BlurIndex { if (indexReader instanceof ExitableReader) { ((ExitableReader) indexReader).reset(); } - return new IndexSearcherClosable(indexReader, _searchThreadPool) { + if (security) { + return getSecureIndexSearcher(indexReader); + } else { + return getInsecureIndexSearcher(indexReader); + } + } + + private IndexSearcherCloseable getSecureIndexSearcher(final IndexReader indexReader) throws IOException { + String readStr = null; + String discoverStr = null; + User user = UserContext.getUser(); + if (user != null) { + Map<String, String> attributes = user.getAttributes(); + if (attributes != null) { + readStr = attributes.get(ACL_READ); + discoverStr = attributes.get(ACL_DISCOVER); + } + } + Collection<String> readAuthorizations = toCollection(readStr); + Collection<String> discoverAuthorizations = toCollection(discoverStr); + return new IndexSearcherCloseableSecureBase(indexReader, _searchThreadPool, _accessControlFactory, + readAuthorizations, discoverAuthorizations, _discoverableFields) { + private boolean _closed; + + @Override + public Directory getDirectory() { + return _directory; + } + + @Override + public synchronized void close() throws IOException { + if (!_closed) { + indexReader.decRef(); + _closed = true; + } else { + // Not really sure why some indexes get closed called twice on them. + // This is in place to log it. + if (LOG.isDebugEnabled()) { + LOG.debug("Searcher already closed [{0}].", new Throwable(), this); + } + } + } + }; + } + + @SuppressWarnings("unchecked") + private Collection<String> toCollection(String aclStr) { + if (aclStr == null) { + return Collections.EMPTY_LIST; + } + Set<String> result = new HashSet<String>(); + for (String s : _commaSplitter.split(aclStr)) { + result.add(s); + } + return result; + } + private IndexSearcherCloseable getInsecureIndexSearcher(final IndexReader indexReader) { + return new IndexSearcherCloseableBase(indexReader, _searchThreadPool) { private boolean _closed; @Override @@ -356,9 +459,9 @@ public class BlurIndexSimpleWriter extends BlurIndex { indexAction.setWritesWaiting(_writesWaiting); waitUntilNotNull(_writer); BlurIndexWriter writer = _writer.get(); - IndexSearcherClosable indexSearcher = null; + IndexSearcherCloseable indexSearcher = null; try { - indexSearcher = getIndexSearcher(); + indexSearcher = getIndexSearcher(false); indexAction.performMutate(indexSearcher, writer); indexAction.doPreCommit(indexSearcher, writer); commit(); @@ -429,13 +532,13 @@ public class BlurIndexSimpleWriter extends BlurIndex { }; final CompressionCodec codec; final CompressionType type; -// if (SnappyCodec.isNativeSnappyLoaded(configuration)) { -// codec = new SnappyCodec(); -// type = CompressionType.BLOCK; -// } else { - codec = new DefaultCodec(); - type = CompressionType.NONE; -// } + // if (SnappyCodec.isNativeSnappyLoaded(configuration)) { + // codec = new SnappyCodec(); + // type = CompressionType.BLOCK; + // } else { + codec = new DefaultCodec(); + type = CompressionType.NONE; + // } Writer writer = SequenceFile.createWriter(fileSystem, configuration, path, Text.class, RowMutationWritable.class, type, codec, progress); @@ -473,7 +576,7 @@ public class BlurIndexSimpleWriter extends BlurIndex { private Path _sorted; @Override - public void performMutate(IndexSearcherClosable searcher, IndexWriter writer) throws IOException { + public void performMutate(IndexSearcherCloseable searcher, IndexWriter writer) throws IOException { Configuration configuration = _tableContext.getConfiguration(); SequenceFile.Sorter sorter = new Sorter(fileSystem, Text.class, RowMutationWritable.class, @@ -506,7 +609,7 @@ public class BlurIndexSimpleWriter extends BlurIndex { LOG.info("Shard [{0}/{1}] Id [{2}] Finished applying mutates starting commit.", table, shard, bulkId); } - private void flushMutates(IndexSearcherClosable searcher, IndexWriter writer, List<RowMutation> list) + private void flushMutates(IndexSearcherCloseable searcher, IndexWriter writer, List<RowMutation> list) throws IOException { if (!list.isEmpty()) { List<RowMutation> reduceMutates; @@ -536,7 +639,7 @@ public class BlurIndexSimpleWriter extends BlurIndex { } @Override - public void doPreCommit(IndexSearcherClosable indexSearcher, IndexWriter writer) throws IOException { + public void doPreCommit(IndexSearcherCloseable indexSearcher, IndexWriter writer) throws IOException { } @@ -596,4 +699,50 @@ public class BlurIndexSimpleWriter extends BlurIndex { } } + @Override + public long getRecordCount() throws IOException { + IndexSearcherCloseable searcher = getIndexSearcher(false); + try { + return searcher.getIndexReader().numDocs(); + } finally { + if (searcher != null) { + searcher.close(); + } + } + } + + @Override + public long getRowCount() throws IOException { + IndexSearcherCloseable searcher = getIndexSearcher(false); + try { + return getRowCount(searcher); + } finally { + if (searcher != null) { + searcher.close(); + } + } + } + + protected long getRowCount(IndexSearcherCloseable searcher) throws IOException { + TopDocs topDocs = searcher.search(new TermQuery(BlurUtil.PRIME_DOC_TERM), 1); + return topDocs.totalHits; + } + + @Override + public long getIndexMemoryUsage() throws IOException { + return 0; + } + + @Override + public long getSegmentCount() throws IOException { + IndexSearcherCloseable indexSearcherClosable = getIndexSearcher(false); + try { + IndexReader indexReader = indexSearcherClosable.getIndexReader(); + IndexReaderContext context = indexReader.getContext(); + return context.leaves().size(); + } finally { + indexSearcherClosable.close(); + } + } + } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-core/src/main/java/org/apache/blur/manager/writer/IndexAction.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/IndexAction.java b/blur-core/src/main/java/org/apache/blur/manager/writer/IndexAction.java index 6d56c5a..7c43598 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/IndexAction.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/IndexAction.java @@ -19,14 +19,14 @@ package org.apache.blur.manager.writer; import java.io.IOException; import java.util.concurrent.atomic.AtomicInteger; -import org.apache.blur.server.IndexSearcherClosable; +import org.apache.blur.lucene.search.IndexSearcherCloseable; import org.apache.lucene.index.IndexWriter; public abstract class IndexAction { private AtomicInteger _writesWaiting; - public abstract void doPreCommit(IndexSearcherClosable indexSearcher, IndexWriter writer) throws IOException; + public abstract void doPreCommit(IndexSearcherCloseable indexSearcher, IndexWriter writer) throws IOException; public abstract void doPostCommit(IndexWriter writer) throws IOException; @@ -34,7 +34,7 @@ public abstract class IndexAction { public abstract void doPostRollback(IndexWriter writer) throws IOException; - public abstract void performMutate(IndexSearcherClosable searcher, IndexWriter writer) throws IOException; + public abstract void performMutate(IndexSearcherCloseable searcher, IndexWriter writer) throws IOException; public void setWritesWaiting(AtomicInteger writesWaiting) { _writesWaiting = writesWaiting; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java b/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java index 74363ea..cec713c 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java @@ -33,8 +33,8 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; +import org.apache.blur.lucene.search.IndexSearcherCloseable; import org.apache.blur.manager.BlurPartitioner; -import org.apache.blur.server.IndexSearcherClosable; import org.apache.blur.server.ShardContext; import org.apache.blur.server.TableContext; import org.apache.blur.store.hdfs.HdfsDirectory; @@ -178,7 +178,7 @@ public class IndexImporter extends TimerTask implements Closeable { return new IndexAction() { @Override - public void performMutate(IndexSearcherClosable searcher, IndexWriter writer) throws IOException { + public void performMutate(IndexSearcherCloseable searcher, IndexWriter writer) throws IOException { LOG.info("About to import [{0}] into [{1}/{2}]", directory, _shard, _table); boolean emitDeletes = searcher.getIndexReader().numDocs() != 0; applyDeletes(directory, writer, _shard, emitDeletes); @@ -190,7 +190,7 @@ public class IndexImporter extends TimerTask implements Closeable { } @Override - public void doPreCommit(IndexSearcherClosable indexSearcher, IndexWriter writer) throws IOException { + public void doPreCommit(IndexSearcherCloseable indexSearcher, IndexWriter writer) throws IOException { } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-core/src/main/java/org/apache/blur/manager/writer/MutatableAction.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/MutatableAction.java b/blur-core/src/main/java/org/apache/blur/manager/writer/MutatableAction.java index b5858b4..00c1939 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/MutatableAction.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/MutatableAction.java @@ -34,7 +34,7 @@ import java.util.TreeSet; import java.util.concurrent.TimeUnit; import org.apache.blur.analysis.FieldManager; -import org.apache.blur.server.IndexSearcherClosable; +import org.apache.blur.lucene.search.IndexSearcherCloseable; import org.apache.blur.server.ShardContext; import org.apache.blur.server.TableContext; import org.apache.blur.thrift.BException; @@ -364,7 +364,7 @@ public class MutatableAction extends IndexAction { } @Override - void performAction(IndexSearcherClosable searcher, IndexWriter writer) throws IOException { + void performAction(IndexSearcherCloseable searcher, IndexWriter writer) throws IOException { IterableRow iterableRow = getIterableRow(_rowId, searcher); for (UpdateRowAction action : _actions) { iterableRow = action.performAction(iterableRow); @@ -399,7 +399,7 @@ public class MutatableAction extends IndexAction { } } - private IterableRow getIterableRow(String rowId, IndexSearcherClosable searcher) throws IOException { + private IterableRow getIterableRow(String rowId, IndexSearcherCloseable searcher) throws IOException { IndexReader indexReader = searcher.getIndexReader(); BytesRef rowIdRef = new BytesRef(rowId); List<AtomicReaderTermsEnum> possibleRowIds = new ArrayList<AtomicReaderTermsEnum>(); @@ -540,7 +540,7 @@ public class MutatableAction extends IndexAction { } static abstract class InternalAction { - abstract void performAction(IndexSearcherClosable searcher, IndexWriter writer) throws IOException; + abstract void performAction(IndexSearcherCloseable searcher, IndexWriter writer) throws IOException; } private final List<InternalAction> _actions = new ArrayList<InternalAction>(); @@ -556,7 +556,7 @@ public class MutatableAction extends IndexAction { public void deleteRow(final String rowId) { _actions.add(new InternalAction() { @Override - void performAction(IndexSearcherClosable searcher, IndexWriter writer) throws IOException { + void performAction(IndexSearcherCloseable searcher, IndexWriter writer) throws IOException { writer.deleteDocuments(createRowId(rowId)); _writeRowMeter.mark(); } @@ -566,7 +566,7 @@ public class MutatableAction extends IndexAction { public void replaceRow(final Row row) { _actions.add(new InternalAction() { @Override - void performAction(IndexSearcherClosable searcher, IndexWriter writer) throws IOException { + void performAction(IndexSearcherCloseable searcher, IndexWriter writer) throws IOException { List<List<Field>> docs = RowDocumentUtil.getDocs(row, _fieldManager); Term rowId = createRowId(row.getId()); writer.updateDocuments(rowId, docs); @@ -597,7 +597,7 @@ public class MutatableAction extends IndexAction { } @Override - public void performMutate(IndexSearcherClosable searcher, IndexWriter writer) throws IOException { + public void performMutate(IndexSearcherCloseable searcher, IndexWriter writer) throws IOException { try { for (InternalAction internalAction : _actions) { internalAction.performAction(searcher, writer); @@ -626,7 +626,7 @@ public class MutatableAction extends IndexAction { } @Override - public void doPreCommit(IndexSearcherClosable indexSearcher, IndexWriter writer) { + public void doPreCommit(IndexSearcherCloseable indexSearcher, IndexWriter writer) { } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-core/src/main/java/org/apache/blur/manager/writer/MutationQueueProcessor.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/MutationQueueProcessor.java b/blur-core/src/main/java/org/apache/blur/manager/writer/MutationQueueProcessor.java index 742381a..2a02371 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/MutationQueueProcessor.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/MutationQueueProcessor.java @@ -32,7 +32,7 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.blur.BlurConfiguration; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; -import org.apache.blur.server.IndexSearcherClosable; +import org.apache.blur.lucene.search.IndexSearcherCloseable; import org.apache.blur.server.ShardContext; import org.apache.blur.server.TableContext; import org.apache.blur.thrift.generated.BlurException; @@ -135,7 +135,7 @@ public class MutationQueueProcessor implements Runnable, Closeable { } @Override - public void performMutate(IndexSearcherClosable searcher, IndexWriter writer) throws IOException { + public void performMutate(IndexSearcherCloseable searcher, IndexWriter writer) throws IOException { List<RowMutation> lst = new ArrayList<RowMutation>(); while (shouldContinueProcessing()) { if (_queue.drainTo(lst, _maxQueueBatch) > 0) { @@ -166,7 +166,7 @@ public class MutationQueueProcessor implements Runnable, Closeable { } @Override - public void doPreCommit(IndexSearcherClosable indexSearcher, IndexWriter writer) throws IOException { + public void doPreCommit(IndexSearcherCloseable indexSearcher, IndexWriter writer) throws IOException { } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-core/src/main/java/org/apache/blur/server/IndexSearcherClosable.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/server/IndexSearcherClosable.java b/blur-core/src/main/java/org/apache/blur/server/IndexSearcherClosable.java deleted file mode 100644 index 673d47f..0000000 --- a/blur-core/src/main/java/org/apache/blur/server/IndexSearcherClosable.java +++ /dev/null @@ -1,76 +0,0 @@ -package org.apache.blur.server; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -import java.io.Closeable; -import java.io.IOException; -import java.util.List; -import java.util.concurrent.ExecutorService; - -import org.apache.blur.trace.Trace; -import org.apache.blur.trace.Tracer; -import org.apache.lucene.index.AtomicReaderContext; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.search.CollectionTerminatedException; -import org.apache.lucene.search.Collector; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.Weight; -import org.apache.lucene.store.Directory; - -public abstract class IndexSearcherClosable extends IndexSearcher implements Closeable { - - public IndexSearcherClosable(IndexReader r, ExecutorService executor) { - super(r, executor); - } - - public abstract Directory getDirectory(); - - @Override - public abstract void close() throws IOException; - - protected void search(List<AtomicReaderContext> leaves, Weight weight, Collector collector) throws IOException { - - // TODO: should we make this - // threaded...? the Collector could be sync'd? - // always use single thread: - for (AtomicReaderContext ctx : leaves) { // search each subreader - Tracer trace = Trace.trace("search - internal", Trace.param("AtomicReader", ctx.reader())); - try { - try { - collector.setNextReader(ctx); - } catch (CollectionTerminatedException e) { - // there is no doc of interest in this reader context - // continue with the following leaf - continue; - } - Scorer scorer = weight.scorer(ctx, !collector.acceptsDocsOutOfOrder(), true, ctx.reader().getLiveDocs()); - if (scorer != null) { - try { - scorer.score(collector); - } catch (CollectionTerminatedException e) { - // collection was terminated prematurely - // continue with the following leaf - } - } - } finally { - trace.done(); - } - } - } - -} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-core/src/main/java/org/apache/blur/server/IndexSearcherCloseableBase.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/server/IndexSearcherCloseableBase.java b/blur-core/src/main/java/org/apache/blur/server/IndexSearcherCloseableBase.java new file mode 100644 index 0000000..053d1f2 --- /dev/null +++ b/blur-core/src/main/java/org/apache/blur/server/IndexSearcherCloseableBase.java @@ -0,0 +1,75 @@ +package org.apache.blur.server; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +import java.io.IOException; +import java.util.List; +import java.util.concurrent.ExecutorService; + +import org.apache.blur.lucene.search.IndexSearcherCloseable; +import org.apache.blur.trace.Trace; +import org.apache.blur.trace.Tracer; +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.CollectionTerminatedException; +import org.apache.lucene.search.Collector; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.Weight; +import org.apache.lucene.store.Directory; + +public abstract class IndexSearcherCloseableBase extends IndexSearcher implements IndexSearcherCloseable { + + public IndexSearcherCloseableBase(IndexReader r, ExecutorService executor) { + super(r, executor); + } + + public abstract Directory getDirectory(); + + @Override + public abstract void close() throws IOException; + + protected void search(List<AtomicReaderContext> leaves, Weight weight, Collector collector) throws IOException { + // TODO: should we make this + // threaded...? the Collector could be sync'd? + // always use single thread: + for (AtomicReaderContext ctx : leaves) { // search each subreader + Tracer trace = Trace.trace("search - internal", Trace.param("AtomicReader", ctx.reader())); + try { + try { + collector.setNextReader(ctx); + } catch (CollectionTerminatedException e) { + // there is no doc of interest in this reader context + // continue with the following leaf + continue; + } + Scorer scorer = weight.scorer(ctx, !collector.acceptsDocsOutOfOrder(), true, ctx.reader().getLiveDocs()); + if (scorer != null) { + try { + scorer.score(collector); + } catch (CollectionTerminatedException e) { + // collection was terminated prematurely + // continue with the following leaf + } + } + } finally { + trace.done(); + } + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-core/src/main/java/org/apache/blur/server/IndexSearcherCloseableSecureBase.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/server/IndexSearcherCloseableSecureBase.java b/blur-core/src/main/java/org/apache/blur/server/IndexSearcherCloseableSecureBase.java new file mode 100644 index 0000000..86ee3ed --- /dev/null +++ b/blur-core/src/main/java/org/apache/blur/server/IndexSearcherCloseableSecureBase.java @@ -0,0 +1,76 @@ +package org.apache.blur.server; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +import java.io.IOException; +import java.util.Collection; +import java.util.Set; +import java.util.concurrent.ExecutorService; + +import lucene.security.index.AccessControlFactory; +import lucene.security.search.SecureIndexSearcher; + +import org.apache.blur.lucene.search.IndexSearcherCloseable; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.store.Directory; + +public abstract class IndexSearcherCloseableSecureBase extends SecureIndexSearcher implements IndexSearcherCloseable { + + public IndexSearcherCloseableSecureBase(IndexReader r, ExecutorService executor, + AccessControlFactory accessControlFactory, Collection<String> readAuthorizations, + Collection<String> discoverAuthorizations, Set<String> discoverableFields) throws IOException { + super(r, executor, accessControlFactory, readAuthorizations, discoverAuthorizations, discoverableFields); + } + + public abstract Directory getDirectory(); + + @Override + public abstract void close() throws IOException; + // + // protected void search(List<AtomicReaderContext> leaves, Weight weight, + // Collector collector) throws IOException { + // // TODO: should we make this + // // threaded...? the Collector could be sync'd? + // // always use single thread: + // for (AtomicReaderContext ctx : leaves) { // search each subreader + // Tracer trace = Trace.trace("search - internal", Trace.param("AtomicReader", + // ctx.reader())); + // try { + // try { + // collector.setNextReader(ctx); + // } catch (CollectionTerminatedException e) { + // // there is no doc of interest in this reader context + // // continue with the following leaf + // continue; + // } + // Scorer scorer = weight.scorer(ctx, !collector.acceptsDocsOutOfOrder(), + // true, ctx.reader().getLiveDocs()); + // if (scorer != null) { + // try { + // scorer.score(collector); + // } catch (CollectionTerminatedException e) { + // // collection was terminated prematurely + // // continue with the following leaf + // } + // } + // } finally { + // trace.done(); + // } + // } + // } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-core/src/main/java/org/apache/blur/server/ShardServerContext.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/server/ShardServerContext.java b/blur-core/src/main/java/org/apache/blur/server/ShardServerContext.java index a605af4..59b142d 100644 --- a/blur-core/src/main/java/org/apache/blur/server/ShardServerContext.java +++ b/blur-core/src/main/java/org/apache/blur/server/ShardServerContext.java @@ -25,6 +25,7 @@ import java.util.concurrent.ConcurrentHashMap; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; +import org.apache.blur.lucene.search.IndexSearcherCloseable; import org.apache.blur.thirdparty.thrift_0_9_0.server.ServerContext; /** @@ -38,7 +39,7 @@ public class ShardServerContext extends BlurServerContext implements ServerConte private static final Log LOG = LogFactory.getLog(ShardServerContext.class); private final static Map<Thread, ShardServerContext> _threadsToContext = new ConcurrentHashMap<Thread, ShardServerContext>(); - private final Map<String, IndexSearcherClosable> _indexSearcherMap = new ConcurrentHashMap<String, IndexSearcherClosable>(); + private final Map<String, IndexSearcherCloseable> _indexSearcherMap = new ConcurrentHashMap<String, IndexSearcherCloseable>(); public ShardServerContext(SocketAddress localSocketAddress, SocketAddress remoteSocketAddress) { super(localSocketAddress, remoteSocketAddress); @@ -85,8 +86,8 @@ public class ShardServerContext extends BlurServerContext implements ServerConte * Resets the {@link ShardServerContext} by closing the searchers. */ public void reset() { - Collection<IndexSearcherClosable> values = _indexSearcherMap.values(); - for (IndexSearcherClosable indexSearcherClosable : values) { + Collection<IndexSearcherCloseable> values = _indexSearcherMap.values(); + for (IndexSearcherCloseable indexSearcherClosable : values) { LOG.debug("Closing [{0}]", indexSearcherClosable); closeQuietly(indexSearcherClosable); } @@ -104,17 +105,17 @@ public class ShardServerContext extends BlurServerContext implements ServerConte } /** - * Gets the cached {@link IndexSearcherClosable} (if any) for the given table + * Gets the cached {@link IndexSearcherCloseable} (if any) for the given table * and shard. * * @param table * the stable name. * @param shard * the shard name. - * @return the {@link IndexSearcherClosable} or null if not present. + * @return the {@link IndexSearcherCloseable} or null if not present. */ - public IndexSearcherClosable getIndexSearcherClosable(String table, String shard) { - IndexSearcherClosable indexSearcherClosable = _indexSearcherMap.get(getKey(table, shard)); + public IndexSearcherCloseable getIndexSearcherClosable(String table, String shard) { + IndexSearcherCloseable indexSearcherClosable = _indexSearcherMap.get(getKey(table, shard)); if (indexSearcherClosable != null) { LOG.debug("Using cached searcher [{0}] for table [{1}] shard [{2}]", indexSearcherClosable, table, shard); } @@ -130,11 +131,11 @@ public class ShardServerContext extends BlurServerContext implements ServerConte * @param shard * the shard name. * @param searcher - * the {@link IndexSearcherClosable}. + * the {@link IndexSearcherCloseable}. * @throws IOException */ - public void setIndexSearcherClosable(String table, String shard, IndexSearcherClosable searcher) throws IOException { - IndexSearcherClosable indexSearcherClosable = _indexSearcherMap.put(getKey(table, shard), searcher); + public void setIndexSearcherClosable(String table, String shard, IndexSearcherCloseable searcher) throws IOException { + IndexSearcherCloseable indexSearcherClosable = _indexSearcherMap.put(getKey(table, shard), searcher); if (indexSearcherClosable != null && searcher != indexSearcherClosable) { indexSearcherClosable.close(); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-core/src/main/java/org/apache/blur/server/TableContext.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/server/TableContext.java b/blur-core/src/main/java/org/apache/blur/server/TableContext.java index 54186e8..ef50bff 100644 --- a/blur-core/src/main/java/org/apache/blur/server/TableContext.java +++ b/blur-core/src/main/java/org/apache/blur/server/TableContext.java @@ -28,7 +28,9 @@ import static org.apache.blur.utils.BlurConstants.SUPER; import java.io.IOException; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; +import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; import java.util.Map.Entry; import java.util.Set; @@ -37,6 +39,9 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; +import lucene.security.index.AccessControlFactory; +import lucene.security.index.FilterAccessControlFactory; + import org.apache.blur.BlurConfiguration; import org.apache.blur.analysis.FieldManager; import org.apache.blur.analysis.FieldTypeDefinition; @@ -103,6 +108,8 @@ public class TableContext implements Cloneable { private FieldManager _fieldManager; private BlurConfiguration _blurConfiguration; private ReadInterceptor _readInterceptor; + private AccessControlFactory _accessControlFactory; + private Set<String> _discoverableFields; protected TableContext() { @@ -164,6 +171,13 @@ public class TableContext implements Cloneable { tableContext._defaultPrimeDocTerm = new Term(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE); tableContext._defaultScoreType = ScoreType.SUPER; + // TODO make configurable + tableContext._discoverableFields = new HashSet<String>(Arrays.asList(BlurConstants.ROW_ID, BlurConstants.RECORD_ID, + BlurConstants.FAMILY)); + + // TODO make configurable + tableContext._accessControlFactory = new FilterAccessControlFactory(); + boolean strict = tableDescriptor.isStrictTypes(); String defaultMissingFieldType = tableDescriptor.getDefaultMissingFieldType(); boolean defaultMissingFieldLessIndexing = tableDescriptor.isDefaultMissingFieldLessIndexing(); @@ -465,4 +479,12 @@ public class TableContext implements Cloneable { } } } + + public Set<String> getDiscoverableFields() { + return _discoverableFields; + } + + public AccessControlFactory getAccessControlFactory() { + return _accessControlFactory; + } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-core/src/main/java/org/apache/blur/utils/GCWatcherJdk7.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/utils/GCWatcherJdk7.java b/blur-core/src/main/java/org/apache/blur/utils/GCWatcherJdk7.java index 5164167..f9bfd65 100644 --- a/blur-core/src/main/java/org/apache/blur/utils/GCWatcherJdk7.java +++ b/blur-core/src/main/java/org/apache/blur/utils/GCWatcherJdk7.java @@ -68,27 +68,32 @@ public class GCWatcherJdk7 { public void handleNotification(Notification notification, Object bean) { GarbageCollectorMXBean garbageCollectorMXBean = (GarbageCollectorMXBean) bean; GcInfo gcInfo = getGcInfo(garbageCollectorMXBean); - long startTime = gcInfo.getStartTime(); - long endTime = gcInfo.getEndTime(); - Map<String, MemoryUsage> usageBeforeGc = gcInfo.getMemoryUsageBeforeGc(); - Map<String, MemoryUsage> usageAfterGc = gcInfo.getMemoryUsageAfterGc(); - long usedBefore = getTotal(usageBeforeGc); - long usedAfter = getTotal(usageAfterGc); - long totalTime = endTime - startTime; - long totalSize = usedBefore - usedAfter; - if (totalTime >= _1_SECOND) { - LOG.info("GC event totalTime spent in GC [{0} ms] collected [{1} bytes]", totalTime, totalSize); - } - _gcTimes.update(totalTime, TimeUnit.MILLISECONDS); - - MemoryUsage heapMemoryUsage = _memoryMXBean.getHeapMemoryUsage(); - long max = heapMemoryUsage.getMax(); - long used = heapMemoryUsage.getUsed(); - long upperLimit = (long) (max * _ratio); - if (used > upperLimit) { - LOG.error("----- WARNING !!!! - Heap used [{0}] over limit of [{1}], taking action to avoid an OOM error.", - used, upperLimit); - takeAction(); + if (gcInfo != null) { + long startTime = gcInfo.getStartTime(); + long endTime = gcInfo.getEndTime(); + Map<String, MemoryUsage> usageBeforeGc = gcInfo.getMemoryUsageBeforeGc(); + Map<String, MemoryUsage> usageAfterGc = gcInfo.getMemoryUsageAfterGc(); + long usedBefore = getTotal(usageBeforeGc); + long usedAfter = getTotal(usageAfterGc); + long totalTime = endTime - startTime; + long totalSize = usedBefore - usedAfter; + if (totalTime >= _1_SECOND) { + LOG.info("GC event totalTime spent in GC [{0} ms] collected [{1} bytes]", totalTime, totalSize); + } + _gcTimes.update(totalTime, TimeUnit.MILLISECONDS); + + MemoryUsage heapMemoryUsage = _memoryMXBean.getHeapMemoryUsage(); + long max = heapMemoryUsage.getMax(); + long used = heapMemoryUsage.getUsed(); + long upperLimit = (long) (max * _ratio); + if (used > upperLimit) { + LOG.error( + "----- WARNING !!!! - Heap used [{0}] over limit of [{1}], taking action to avoid an OOM error.", + used, upperLimit); + takeAction(); + } + } else { + LOG.warn("GCInfo was null. Cannot report on GC activity."); } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-core/src/test/java/org/apache/blur/MiniCluster.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/MiniCluster.java b/blur-core/src/test/java/org/apache/blur/MiniCluster.java index 5c30019..796b7f4 100644 --- a/blur-core/src/test/java/org/apache/blur/MiniCluster.java +++ b/blur-core/src/test/java/org/apache/blur/MiniCluster.java @@ -348,8 +348,8 @@ public class MiniCluster { List<String> command = new ArrayList<String>(); command.add(javaHome + "/bin/java"); - command.add("-Xmx256m"); - command.add("-Xms256m"); + command.add("-Xmx512m"); + command.add("-Xms512m"); command.add("-cp"); command.add(classPath); command.add(ExternalThriftServer.class.getName()); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java b/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java index 56784d7..e3ecbe3 100644 --- a/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java +++ b/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java @@ -34,10 +34,11 @@ import java.util.concurrent.CancellationException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.blur.lucene.search.IndexSearcherCloseable; +import org.apache.blur.lucene.search.IndexSearcherCloseableUtil; import org.apache.blur.manager.IndexServer; import org.apache.blur.manager.writer.BlurIndex; import org.apache.blur.manager.writer.IndexAction; -import org.apache.blur.server.IndexSearcherClosable; import org.apache.blur.server.ShardContext; import org.apache.blur.server.TableContext; import org.apache.blur.server.TableContextFactory; @@ -54,6 +55,7 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.Version; @@ -401,20 +403,9 @@ public class ShardCommandManagerTest { } @Override - public IndexSearcherClosable getIndexSearcher() throws IOException { + public IndexSearcherCloseable getIndexSearcher() throws IOException { IndexReader reader = getEmtpyReader(); - return new IndexSearcherClosable(reader, null) { - - @Override - public Directory getDirectory() { - return getEmtpyDirectory(); - } - - @Override - public void close() throws IOException { - - } - }; + return IndexSearcherCloseableUtil.wrap(new IndexSearcher(reader)); } @Override http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexSimpleWriterTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexSimpleWriterTest.java b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexSimpleWriterTest.java index c2d3b15..9075618 100644 --- a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexSimpleWriterTest.java +++ b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexSimpleWriterTest.java @@ -33,7 +33,7 @@ import java.util.concurrent.ExecutorService; import org.apache.blur.BlurConfiguration; import org.apache.blur.concurrent.Executors; -import org.apache.blur.server.IndexSearcherClosable; +import org.apache.blur.lucene.search.IndexSearcherCloseable; import org.apache.blur.server.ShardContext; import org.apache.blur.server.TableContext; import org.apache.blur.store.hdfs.BlurLockFactory; @@ -56,9 +56,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.store.LockFactory; -import org.json.JSONException; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -157,7 +154,7 @@ public class BlurIndexSimpleWriterTest { public void testRollbackAndReopen() throws IOException { setupWriter(_configuration); { - IndexSearcherClosable searcher = _writer.getIndexSearcher(); + IndexSearcherCloseable searcher = _writer.getIndexSearcher(); IndexReader reader = searcher.getIndexReader(); assertEquals(0, reader.numDocs()); searcher.close(); @@ -172,7 +169,7 @@ public class BlurIndexSimpleWriterTest { // do nothing } { - IndexSearcherClosable searcher = _writer.getIndexSearcher(); + IndexSearcherCloseable searcher = _writer.getIndexSearcher(); IndexReader reader = searcher.getIndexReader(); assertEquals(0, reader.numDocs()); searcher.close(); @@ -182,7 +179,7 @@ public class BlurIndexSimpleWriterTest { _writer.process(action); { - IndexSearcherClosable searcher = _writer.getIndexSearcher(); + IndexSearcherCloseable searcher = _writer.getIndexSearcher(); IndexReader reader = searcher.getIndexReader(); assertEquals(1, reader.numDocs()); searcher.close(); @@ -215,7 +212,7 @@ public class BlurIndexSimpleWriterTest { MutatableAction action = new MutatableAction(_writer.getShardContext()); action.replaceRow(genRow()); _writer.process(action); - IndexSearcherClosable searcher = _writer.getIndexSearcher(); + IndexSearcherCloseable searcher = _writer.getIndexSearcher(); IndexReader reader = searcher.getIndexReader(); assertEquals(i + 1, reader.numDocs()); searcher.close(); @@ -226,7 +223,7 @@ public class BlurIndexSimpleWriterTest { double seconds = (e - s) / 1000000000.0; double rate = total / seconds; System.out.println("Rate " + rate); - IndexSearcherClosable searcher = _writer.getIndexSearcher(); + IndexSearcherCloseable searcher = _writer.getIndexSearcher(); IndexReader reader = searcher.getIndexReader(); assertEquals(TEST_NUMBER_WAIT_VISIBLE, reader.numDocs()); searcher.close(); @@ -236,7 +233,7 @@ public class BlurIndexSimpleWriterTest { @Test public void testBlurIndexWriterFaster() throws IOException, InterruptedException { setupWriter(_configuration); - IndexSearcherClosable searcher1 = _writer.getIndexSearcher(); + IndexSearcherCloseable searcher1 = _writer.getIndexSearcher(); IndexReader reader1 = searcher1.getIndexReader(); assertEquals(0, reader1.numDocs()); searcher1.close(); @@ -256,7 +253,7 @@ public class BlurIndexSimpleWriterTest { // refresh once every 25 ms Thread.sleep(1000);// Hack for now _writer.refresh(); - IndexSearcherClosable searcher2 = _writer.getIndexSearcher(); + IndexSearcherCloseable searcher2 = _writer.getIndexSearcher(); IndexReader reader2 = searcher2.getIndexReader(); assertEquals(TEST_NUMBER, reader2.numDocs()); searcher2.close(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-core/src/test/java/org/apache/blur/manager/writer/IndexImporterTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/manager/writer/IndexImporterTest.java b/blur-core/src/test/java/org/apache/blur/manager/writer/IndexImporterTest.java index 9e83196..6f2eee7 100644 --- a/blur-core/src/test/java/org/apache/blur/manager/writer/IndexImporterTest.java +++ b/blur-core/src/test/java/org/apache/blur/manager/writer/IndexImporterTest.java @@ -30,7 +30,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.blur.analysis.FieldManager; -import org.apache.blur.server.IndexSearcherClosable; +import org.apache.blur.lucene.search.IndexSearcherCloseable; +import org.apache.blur.lucene.search.IndexSearcherCloseableUtil; import org.apache.blur.server.ShardContext; import org.apache.blur.server.TableContext; import org.apache.blur.store.buffer.BufferStore; @@ -49,6 +50,7 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.TieredMergePolicy; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.store.Directory; import org.junit.After; import org.junit.Before; @@ -139,18 +141,7 @@ public class IndexImporterTest { @Override public void process(IndexAction indexAction) throws IOException { final DirectoryReader reader = DirectoryReader.open(mainDirectory); - IndexSearcherClosable searcherClosable = new IndexSearcherClosable(reader, null) { - - @Override - public Directory getDirectory() { - return mainDirectory; - } - - @Override - public void close() throws IOException { - reader.close(); - } - }; + IndexSearcherCloseable searcherClosable = IndexSearcherCloseableUtil.wrap(new IndexSearcher(reader)); try { indexAction.performMutate(searcherClosable, _mainWriter); indexAction.doPreCommit(searcherClosable, _mainWriter); @@ -179,7 +170,7 @@ public class IndexImporterTest { } @Override - public IndexSearcherClosable getIndexSearcher() throws IOException { + public IndexSearcherCloseable getIndexSearcher() throws IOException { throw new RuntimeException("Not Implemented"); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-core/src/test/java/org/apache/blur/manager/writer/MutatableActionTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/manager/writer/MutatableActionTest.java b/blur-core/src/test/java/org/apache/blur/manager/writer/MutatableActionTest.java index 3f3e993..aa041bc 100644 --- a/blur-core/src/test/java/org/apache/blur/manager/writer/MutatableActionTest.java +++ b/blur-core/src/test/java/org/apache/blur/manager/writer/MutatableActionTest.java @@ -24,7 +24,8 @@ import java.util.ArrayList; import java.util.List; import java.util.Random; -import org.apache.blur.server.IndexSearcherClosable; +import org.apache.blur.lucene.search.IndexSearcherCloseable; +import org.apache.blur.lucene.search.IndexSearcherCloseableUtil; import org.apache.blur.server.ShardContext; import org.apache.blur.server.TableContext; import org.apache.blur.thrift.generated.Column; @@ -113,19 +114,8 @@ public class MutatableActionTest { assertEquals(2, reader.numDocs()); } - private IndexSearcherClosable getSearcher(DirectoryReader reader, final Directory directory) { - return new IndexSearcherClosable(reader, null) { - - @Override - public Directory getDirectory() { - return directory; - } - - @Override - public void close() throws IOException { - - } - }; + private IndexSearcherCloseable getSearcher(DirectoryReader reader, final Directory directory) { + return IndexSearcherCloseableUtil.wrap(new IndexSearcher(reader)); } @Test
