Updated Branches: refs/heads/master d54a7fc81 -> 5373a5577
BLUR:148 warm up new segments after NRT reload Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/5373a557 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/5373a557 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/5373a557 Branch: refs/heads/master Commit: 5373a55775e4e2a3b9c7ccfff88a345300646c5b Parents: d54a7fc Author: Gagan <[email protected]> Authored: Tue Jun 25 09:46:37 2013 +0530 Committer: Gagan <[email protected]> Committed: Tue Jun 25 09:46:37 2013 +0530 ---------------------------------------------------------------------- .../blur/manager/writer/BlurNRTIndex.java | 2 +- .../blur/manager/writer/FieldBasedWarmer.java | 79 ++++++++------------ 2 files changed, 31 insertions(+), 50 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5373a557/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java index 2b350fa..4b0bb20 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java @@ -85,7 +85,7 @@ public class BlurNRTIndex extends BlurIndex { conf.setWriteLockTimeout(TimeUnit.MINUTES.toMillis(5)); conf.setSimilarity(_tableContext.getSimilarity()); conf.setIndexDeletionPolicy(_tableContext.getIndexDeletionPolicy()); - conf.setMergedSegmentWarmer(new FieldBasedWarmer(shardContext)); + conf.setMergedSegmentWarmer(new FieldBasedWarmer(shardContext, _isClosed)); TieredMergePolicy mergePolicy = (TieredMergePolicy) conf.getMergePolicy(); mergePolicy.setUseCompoundFile(false); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5373a557/blur-core/src/main/java/org/apache/blur/manager/writer/FieldBasedWarmer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/FieldBasedWarmer.java b/blur-core/src/main/java/org/apache/blur/manager/writer/FieldBasedWarmer.java index 7b275d2..63d687d 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/FieldBasedWarmer.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/FieldBasedWarmer.java @@ -18,13 +18,15 @@ package org.apache.blur.manager.writer; */ import java.io.IOException; import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; +import org.apache.blur.lucene.warmup.IndexTracerResult; +import org.apache.blur.lucene.warmup.IndexWarmup; import org.apache.blur.server.ShardContext; -import org.apache.blur.server.TableContext; import org.apache.blur.thrift.generated.ColumnPreCache; -import org.apache.blur.thrift.generated.TableDescriptor; import org.apache.lucene.index.AtomicReader; import org.apache.lucene.index.Fields; import org.apache.lucene.index.IndexWriter.IndexReaderWarmer; @@ -33,62 +35,41 @@ public class FieldBasedWarmer extends IndexReaderWarmer { private static final Log LOG = LogFactory.getLog(FieldBasedWarmer.class); - private List<String> preCacheCols; - - public FieldBasedWarmer(ShardContext shardContext) { - this(shardContext.getTableContext()); - } - - public FieldBasedWarmer(TableContext tableContext) { - this(tableContext.getDescriptor()); - } - - public FieldBasedWarmer(TableDescriptor tableDescriptor) { - this(tableDescriptor.getColumnPreCache()); - } - - public FieldBasedWarmer(ColumnPreCache columnPreCache) { - this(columnPreCache == null ? null : columnPreCache.getPreCacheCols()); - } - - public FieldBasedWarmer(List<String> preCacheCols) { - this.preCacheCols = preCacheCols; + private ShardContext shardContext; + private AtomicBoolean isClosed; + public FieldBasedWarmer(ShardContext shardContext, AtomicBoolean isClosed) { + this.isClosed = isClosed; + this.shardContext = shardContext; } @Override public void warm(AtomicReader reader) throws IOException { + ColumnPreCache columnPreCache = shardContext.getTableContext().getDescriptor().getColumnPreCache(); + List<String> preCacheCols = columnPreCache == null ? null : columnPreCache.getPreCacheCols(); + int maxSampleSize = 1000; + IndexWarmup indexWarmup = new IndexWarmup(isClosed, maxSampleSize); + String context = shardContext.getTableContext().getTable() + "/" + shardContext.getShard(); + Map<String, List<IndexTracerResult>> sampleIndex = indexWarmup.sampleIndex(reader, context); if (preCacheCols != null) { - warm(reader, preCacheCols); + warm(reader, columnPreCache.preCacheCols, indexWarmup, sampleIndex, context, isClosed); } else { Fields fields = reader.fields(); - warm(reader, fields); + warm(reader, fields, indexWarmup, sampleIndex, context, isClosed); } } - private void warm(AtomicReader reader, Iterable<String> fieldNames) throws IOException { - // for (String field : fieldNames) { - // LOG.debug("Warming field [{0}] in reader [{1}]", field, reader); - // - // AtomicReaderContext context = reader.getContext(); - // context. - // - // Fields fields = reader.fields(); - // Terms terms = fields.terms(field); - // TermsEnum termsEnum = terms.iterator(null); - // BytesRef ref = null; - // Term term = new Term(field); - // while ((ref = termsEnum.next()) != null) { - // term.set(field, ref); - // DocsAndPositionsEnum termPositionsEnum = reader.termPositionsEnum(term); - // if (termPositionsEnum != null) { - // while (termPositionsEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { - // int freq = termPositionsEnum.freq(); - // for (int i = 0; i < freq; i++) { - // termPositionsEnum.nextPosition(); - // } - // } - // } - // } - // } + private void warm(AtomicReader reader, Iterable<String> preCacheCols, IndexWarmup indexWarmup, + Map<String, List<IndexTracerResult>> sampleIndex, String context, AtomicBoolean isClosed) { + for (String field : preCacheCols) { + try { + indexWarmup.warm(reader, sampleIndex, field, context); + } catch (IOException e) { + LOG.error("Context [{0}] unknown error trying to warmup the [{1}] field", e, context, field); + } + if (isClosed.get()) { + LOG.info("Context [{0}] index closed", context); + return; + } + } } }
