Updated Branches: refs/heads/apache-blur-0.2 53c40727c -> 2905d33cd
Adding more tracing to isolate where the fetch slow down occurs. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/a1bda375 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/a1bda375 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/a1bda375 Branch: refs/heads/apache-blur-0.2 Commit: a1bda3756f4db028abf63ecd1f5a2ef86be73ba0 Parents: 53c4072 Author: Aaron McCurry <[email protected]> Authored: Thu Nov 28 09:34:42 2013 -0500 Committer: Aaron McCurry <[email protected]> Committed: Thu Nov 28 09:34:42 2013 -0500 ---------------------------------------------------------------------- .../org/apache/blur/manager/IndexManager.java | 5 +- .../manager/indexserver/LocalIndexServer.java | 19 +++-- .../java/org/apache/blur/utils/BlurUtil.java | 10 +++ .../apache/blur/manager/IndexManagerTest.java | 74 +++++++++++++++++++- 4 files changed, 102 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a1bda375/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java index 25b1a8f..8804697 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java +++ b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java @@ -605,7 +605,10 @@ public class IndexManager { } else { fetchResult.exists = true; fetchResult.deleted = false; - String rowId = getRowId(reader, docId); + String rowId = selector.getRowId(); + if (rowId == null) { + rowId = getRowId(reader, docId); + } Term term = new Term(ROW_ID, rowId); if (returnIdsOnly) { int recordCount = BlurUtil.countDocuments(reader, term); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a1bda375/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java b/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java index 5e42c23..51a7468 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java +++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java @@ -52,6 +52,7 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.store.Directory; import org.apache.lucene.store.MMapDirectory; +import org.apache.lucene.store.RAMDirectory; import com.google.common.io.Closer; @@ -65,15 +66,20 @@ public class LocalIndexServer extends AbstractIndexServer { private final ExecutorService _searchExecutor; private final TableContext _tableContext; private final Closer _closer; + private final boolean _ramDir; public LocalIndexServer(TableDescriptor tableDescriptor) throws IOException { + this(tableDescriptor, false); + } + + public LocalIndexServer(TableDescriptor tableDescriptor, boolean ramDir) throws IOException { _closer = Closer.create(); _tableContext = TableContext.create(tableDescriptor); _mergeScheduler = _closer.register(new SharedMergeScheduler(3)); _gc = _closer.register(new DirectoryReferenceFileGC()); _searchExecutor = Executors.newCachedThreadPool(); _closer.register(new CloseableExecutorService(_searchExecutor)); - + _ramDir = ramDir; getIndexes(_tableContext.getTable()); } @@ -129,10 +135,15 @@ public class LocalIndexServer extends AbstractIndexServer { Map<String, BlurIndex> shards = new ConcurrentHashMap<String, BlurIndex>(); int shardCount = _tableContext.getDescriptor().getShardCount(); for (int i = 0; i < shardCount; i++) { + Directory directory; String shardName = BlurUtil.getShardName(BlurConstants.SHARD_PREFIX, i); - File file = new File(tableFile, shardName); - file.mkdirs(); - MMapDirectory directory = new MMapDirectory(file); + if (_ramDir) { + directory = new RAMDirectory(); + } else { + File file = new File(tableFile, shardName); + file.mkdirs(); + directory = new MMapDirectory(file); + } if (!DirectoryReader.indexExists(directory)) { new IndexWriter(directory, new IndexWriterConfig(LUCENE_VERSION, new KeywordAnalyzer())).close(); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a1bda375/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java b/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java index 1b69d35..4f2e830 100644 --- a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java +++ b/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java @@ -759,6 +759,7 @@ public class BlurUtil { public static List<Document> fetchDocuments(IndexReader reader, Term term, ResetableDocumentStoredFieldVisitor fieldSelector, Selector selector, int maxHeap, String context) throws IOException { + Tracer trace = Trace.trace("search for docs to fetch"); IndexSearcher indexSearcher = new IndexSearcher(reader); int docFreq = reader.docFreq(term); @@ -790,6 +791,10 @@ public class BlurUtil { topDocs = null; } + trace.done(); + + Tracer trace2 = Trace.trace("fetching doc from index"); + long readTime = 0; int totalHeap = 0; for (int i = start; i < end; i++) { if (i >= totalHits) { @@ -801,12 +806,17 @@ public class BlurUtil { break; } int doc = scoreDocs.get(i).doc; + long s = System.nanoTime(); indexSearcher.doc(doc, fieldSelector); + long e = System.nanoTime(); + readTime += (e - s); docs.add(fieldSelector.getDocument()); int heapSize = fieldSelector.getSize(); totalHeap += heapSize; fieldSelector.reset(); } + System.out.println("Read time " + readTime); + trace2.done(); return docs; } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a1bda375/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java b/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java index 48ccab3..47342e2 100644 --- a/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java +++ b/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java @@ -42,7 +42,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Random; import java.util.TreeSet; +import java.util.UUID; import java.util.concurrent.atomic.AtomicLongArray; import org.apache.blur.BlurConfiguration; @@ -64,10 +66,14 @@ import org.apache.blur.thrift.generated.Record; import org.apache.blur.thrift.generated.RecordMutation; import org.apache.blur.thrift.generated.Row; import org.apache.blur.thrift.generated.RowMutation; +import org.apache.blur.thrift.generated.RowMutationType; import org.apache.blur.thrift.generated.Schema; import org.apache.blur.thrift.generated.ScoreType; import org.apache.blur.thrift.generated.Selector; import org.apache.blur.thrift.generated.TableDescriptor; +import org.apache.blur.trace.Trace; +import org.apache.blur.trace.TraceCollector; +import org.apache.blur.trace.TraceReporter; import org.apache.blur.utils.BlurConstants; import org.apache.blur.utils.BlurIterator; import org.apache.blur.utils.BlurUtil; @@ -102,7 +108,7 @@ public class IndexManagerTest { tableDescriptor.setTableUri(file.toURI().toString()); tableDescriptor.putToTableProperties("blur.shard.time.between.refreshs", Long.toString(100)); tableDescriptor.setShardCount(1); - server = new LocalIndexServer(tableDescriptor); + server = new LocalIndexServer(tableDescriptor, true); BlurFilterCache filterCache = new DefaultBlurFilterCache(new BlurConfiguration()); long statusCleanupTimerDelay = 1000; @@ -267,6 +273,72 @@ public class IndexManagerTest { } @Test + public void testMutationReplaceLargeRow() throws Exception { + String rowId = "largerow"; + indexManager.mutate(getLargeRow(rowId)); + TraceReporter oldReporter = Trace.getReporter(); + Trace.setReporter(new TraceReporter(new BlurConfiguration()) { + + @Override + public void close() throws IOException { + + } + + @Override + public void report(TraceCollector collector) { + System.out.println(collector.toJson()); + } + }); + + Trace.setupTrace(rowId); + + Selector selector = new Selector().setRowId(rowId); + FetchResult fetchResult = new FetchResult(); + long s = System.nanoTime(); + indexManager.fetchRow(TABLE, selector, fetchResult); + long e = System.nanoTime(); + assertNotNull(fetchResult.rowResult.row); + Trace.tearDownTrace(); + System.out.println((e - s) / 1000000.0); + + Trace.setReporter(oldReporter); + + } + + private RowMutation getLargeRow(String rowId) { + RowMutation rowMutation = new RowMutation(); + rowMutation.setTable(TABLE); + rowMutation.setRowId(rowId); + rowMutation.setWal(true); + rowMutation.setWaitToBeVisible(true); + rowMutation.setRecordMutations(getRecordMutations(10000)); + rowMutation.setRowMutationType(RowMutationType.REPLACE_ROW); + return rowMutation; + } + + private List<RecordMutation> getRecordMutations(int count) { + List<RecordMutation> mutations = new ArrayList<RecordMutation>(); + for (int i = 0; i < count; i++) { + mutations.add(getRecordMutation()); + } + return mutations; + } + + private RecordMutation getRecordMutation() { + RecordMutation mutation = new RecordMutation(); + mutation.setRecordMutationType(REPLACE_ENTIRE_RECORD); + Record record = new Record(); + record.setFamily(FAMILY); + record.setRecordId(UUID.randomUUID().toString()); + Random random = new Random(); + for (int i = 0; i < 10; i++) { + record.addToColumns(new Column("col" + i, "value" + random.nextLong())); + } + mutation.setRecord(record); + return mutation; + } + + @Test public void testFetchRowByRowIdHighlighting() throws Exception { Selector selector = new Selector().setRowId("row-6"); HighlightOptions highlightOptions = new HighlightOptions();
