Fixed BLUR-203
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/71b72250 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/71b72250 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/71b72250 Branch: refs/heads/master Commit: 71b72250c0dd93cf96522c6cba5c4bc9a431ecfc Parents: 54bb4e2 Author: Aaron McCurry <[email protected]> Authored: Mon Aug 12 14:10:25 2013 -0400 Committer: Aaron McCurry <[email protected]> Committed: Mon Aug 12 14:10:25 2013 -0400 ---------------------------------------------------------------------- .../blur/manager/writer/BlurNRTIndex.java | 3 +- .../blur/manager/writer/FieldBasedWarmer.java | 34 +++++++++++--------- .../apache/blur/lucene/warmup/IndexWarmup.java | 10 ++++-- 3 files changed, 27 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/71b72250/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java index 993074a..811d265 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java @@ -34,6 +34,7 @@ import org.apache.blur.log.LogFactory; import org.apache.blur.lucene.store.refcounter.DirectoryReferenceCounter; import org.apache.blur.lucene.store.refcounter.DirectoryReferenceFileGC; import org.apache.blur.lucene.store.refcounter.IndexInputCloser; +import org.apache.blur.lucene.warmup.IndexWarmupIndexDeletionPolicy; import org.apache.blur.lucene.warmup.TraceableDirectory; import org.apache.blur.server.IndexSearcherClosable; import org.apache.blur.server.IndexSearcherClosableNRT; @@ -88,7 +89,7 @@ public class BlurNRTIndex extends BlurIndex { IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, analyzer); conf.setWriteLockTimeout(TimeUnit.MINUTES.toMillis(5)); conf.setSimilarity(_tableContext.getSimilarity()); - conf.setIndexDeletionPolicy(_tableContext.getIndexDeletionPolicy()); + conf.setIndexDeletionPolicy(new IndexWarmupIndexDeletionPolicy(_tableContext.getIndexDeletionPolicy())); conf.setMergedSegmentWarmer(new FieldBasedWarmer(shardContext, _isClosed)); TieredMergePolicy mergePolicy = (TieredMergePolicy) conf.getMergePolicy(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/71b72250/blur-core/src/main/java/org/apache/blur/manager/writer/FieldBasedWarmer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/FieldBasedWarmer.java b/blur-core/src/main/java/org/apache/blur/manager/writer/FieldBasedWarmer.java index 63d687d..822b8d4 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/FieldBasedWarmer.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/FieldBasedWarmer.java @@ -37,16 +37,17 @@ public class FieldBasedWarmer extends IndexReaderWarmer { private ShardContext shardContext; private AtomicBoolean isClosed; + public FieldBasedWarmer(ShardContext shardContext, AtomicBoolean isClosed) { - this.isClosed = isClosed; - this.shardContext = shardContext; + this.isClosed = isClosed; + this.shardContext = shardContext; } @Override public void warm(AtomicReader reader) throws IOException { - ColumnPreCache columnPreCache = shardContext.getTableContext().getDescriptor().getColumnPreCache(); + ColumnPreCache columnPreCache = shardContext.getTableContext().getDescriptor().getColumnPreCache(); List<String> preCacheCols = columnPreCache == null ? null : columnPreCache.getPreCacheCols(); - int maxSampleSize = 1000; + int maxSampleSize = 1000; IndexWarmup indexWarmup = new IndexWarmup(isClosed, maxSampleSize); String context = shardContext.getTableContext().getTable() + "/" + shardContext.getShard(); Map<String, List<IndexTracerResult>> sampleIndex = indexWarmup.sampleIndex(reader, context); @@ -59,17 +60,18 @@ public class FieldBasedWarmer extends IndexReaderWarmer { } private void warm(AtomicReader reader, Iterable<String> preCacheCols, IndexWarmup indexWarmup, - Map<String, List<IndexTracerResult>> sampleIndex, String context, AtomicBoolean isClosed) { - for (String field : preCacheCols) { - try { - indexWarmup.warm(reader, sampleIndex, field, context); - } catch (IOException e) { - LOG.error("Context [{0}] unknown error trying to warmup the [{1}] field", e, context, field); - } - if (isClosed.get()) { - LOG.info("Context [{0}] index closed", context); - return; - } - } + Map<String, List<IndexTracerResult>> sampleIndex, String context, AtomicBoolean isClosed) { + for (String field : preCacheCols) { + try { + indexWarmup.warm(reader, sampleIndex, field, context); + } catch (IOException e) { + LOG.error("Context [{0}] unknown error trying to warmup the [{1}] field", e, context, field); + LOG.error("Current sampleIndex [{0}]",sampleIndex); + } + if (isClosed.get()) { + LOG.info("Context [{0}] index closed", context); + return; + } + } } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/71b72250/blur-store/src/main/java/org/apache/blur/lucene/warmup/IndexWarmup.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/lucene/warmup/IndexWarmup.java b/blur-store/src/main/java/org/apache/blur/lucene/warmup/IndexWarmup.java index 7dbd808..d592cde 100644 --- a/blur-store/src/main/java/org/apache/blur/lucene/warmup/IndexWarmup.java +++ b/blur-store/src/main/java/org/apache/blur/lucene/warmup/IndexWarmup.java @@ -155,8 +155,8 @@ public class IndexWarmup { double seconds = (now - start) / 1000000000.0; double rateMbPerSec = (bytesReadPerPass / seconds) / 1000 / 1000; double complete = (((double) totalLength - (double) length) / (double) totalLength) * 100.0; - LOG.debug("Context [{3}] warming field [{0}] in file [{1}] is [{2}%] complete at rate of [{4} MB/s]", fieldName, - fileName, complete, context, rateMbPerSec); + LOG.debug("Context [{3}] warming field [{0}] in file [{1}] is [{2}%] complete at rate of [{4} MB/s]", + fieldName, fileName, complete, context, rateMbPerSec); start = System.nanoTime(); bytesReadPerPass = 0; if (_isClosed.get()) { @@ -257,7 +257,7 @@ public class IndexWarmup { return results; } IndexTracer tracer = new IndexTracer((TraceableDirectory) directory, _maxSampleSize); - String fileName = SAMPLE_PREFIX + segmentReader.getSegmentName() + SAMPLE_EXT; + String fileName = getSampleFileName(segmentReader.getSegmentName()); List<IndexTracerResult> segmentTraces = new ArrayList<IndexTracerResult>(); if (directory.fileExists(fileName)) { IndexInput input = directory.openInput(fileName, IOContext.READONCE); @@ -289,6 +289,10 @@ public class IndexWarmup { return results; } + public static String getSampleFileName(String segmentName) { + return SAMPLE_PREFIX + segmentName + SAMPLE_EXT; + } + private List<IndexTracerResult> read(IndexInput input) throws IOException { int count = input.readVInt(); List<IndexTracerResult> results = new ArrayList<IndexTracerResult>(count);
