Change merge policy to disable CFS Signed-off-by: Aaron McCurry <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/a6009d6a Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/a6009d6a Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/a6009d6a Branch: refs/heads/0.3.0-lucene-upgrade Commit: a6009d6a85129e65c6565f51e60de8cbf5060560 Parents: 7d8bc42 Author: Saurabh Gupta <[email protected]> Authored: Thu Oct 31 22:43:18 2013 +0100 Committer: Aaron McCurry <[email protected]> Committed: Sun Nov 3 10:06:39 2013 -0500 ---------------------------------------------------------------------- .../java/org/apache/blur/manager/writer/BlurNRTIndex.java | 10 +++++++--- .../apache/blur/utils/TableShardCountCollapserTest.java | 4 +++- .../main/java/org/apache/blur/mapreduce/BlurReducer.java | 6 +++++- .../org/apache/blur/mapreduce/lib/BlurOutputFormat.java | 4 +++- 4 files changed, 18 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a6009d6a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java index cd7df07..a740686 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java @@ -62,6 +62,7 @@ import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexDeletionPolicy; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.MergePolicy; import org.apache.lucene.index.SnapshotDeletionPolicy; import org.apache.lucene.index.TrackingIndexWriter; import org.apache.lucene.search.ControlledRealTimeReopenThread; @@ -116,7 +117,9 @@ public class BlurNRTIndex extends BlurIndex { } conf.setIndexDeletionPolicy(sdp); conf.setMergedSegmentWarmer(new FieldBasedWarmer(shardContext, _isClosed)); - + MergePolicy mergePolicy = (MergePolicy) conf.getMergePolicy(); + mergePolicy.setNoCFSRatio(0.0); + conf.setMergeScheduler(mergeScheduler.getMergeScheduler()); DirectoryReferenceCounter referenceCounter = new DirectoryReferenceCounter(directory, gc, closer); @@ -166,7 +169,8 @@ public class BlurNRTIndex extends BlurIndex { * @throws IOException */ private Map<String, String> loadExistingSnapshots() throws IOException { - Map<String, String> snapshots = new HashMap<String, String>(); + + Map<String, String> snapshots = new HashMap<String, String>(); FileSystem fileSystem = getFileSystem(); FileStatus[] status = fileSystem.listStatus(getSnapshotsDirectoryPath()); @@ -351,7 +355,7 @@ public class BlurNRTIndex extends BlurIndex { } } _writer.commit(); - IndexCommit indexCommit = snapshotter.snapshot(name); + IndexCommit indexCommit = snapshotter.snapshot(); /* * Persist the snapshots info into a tmp file under the snapshots sub-folder http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a6009d6a/blur-core/src/test/java/org/apache/blur/utils/TableShardCountCollapserTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/utils/TableShardCountCollapserTest.java b/blur-core/src/test/java/org/apache/blur/utils/TableShardCountCollapserTest.java index 5be4457..c2b383c 100644 --- a/blur-core/src/test/java/org/apache/blur/utils/TableShardCountCollapserTest.java +++ b/blur-core/src/test/java/org/apache/blur/utils/TableShardCountCollapserTest.java @@ -35,7 +35,7 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexableField; -import org.apache.lucene.index.TieredMergePolicy; +import org.apache.lucene.index.MergePolicy; import org.apache.lucene.util.Version; import org.junit.Before; import org.junit.Test; @@ -95,6 +95,8 @@ public class TableShardCountCollapserTest { throws IOException { HdfsDirectory hdfsDirectory = new HdfsDirectory(configuration, path); IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_44, new KeywordAnalyzer()); + MergePolicy mergePolicy = (MergePolicy) conf.getMergePolicy(); + mergePolicy.setNoCFSRatio(0.0); IndexWriter indexWriter = new IndexWriter(hdfsDirectory, conf); Partitioner<IntWritable, IntWritable> partitioner = new HashPartitioner<IntWritable, IntWritable>(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a6009d6a/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java ---------------------------------------------------------------------- diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java b/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java index 2e5773b..bf629bd 100644 --- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java +++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java @@ -71,8 +71,8 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.MergePolicy; import org.apache.lucene.index.Term; -import org.apache.lucene.index.TieredMergePolicy; import org.apache.lucene.store.BufferedIndexInput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; @@ -304,6 +304,8 @@ public class BlurReducer extends Reducer<Text, BlurMutate, Text, BlurMutate> { if (optimize) { context.setStatus("Starting Copy-Optimize Phase"); IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, _analyzer); + MergePolicy mergePolicy = (MergePolicy) conf.getMergePolicy(); + mergePolicy.setNoCFSRatio(0.0); long s = System.currentTimeMillis(); IndexWriter writer = new IndexWriter(getBiggerBuffers(destDirectory), conf); writer.addIndexes(reader); @@ -457,6 +459,8 @@ public class BlurReducer extends Reducer<Text, BlurMutate, Text, BlurMutate> { nullCheck(_directory); nullCheck(_analyzer); IndexWriterConfig config = new IndexWriterConfig(LUCENE_VERSION, _analyzer); + MergePolicy mergePolicy = (MergePolicy) config.getMergePolicy(); + mergePolicy.setNoCFSRatio(0.0); config.setSimilarity(new FairSimilarity()); config.setRAMBufferSizeMB(_blurTask.getRamBufferSizeMB()); _writer = new IndexWriter(_directory, config); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a6009d6a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java ---------------------------------------------------------------------- diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java index 725be30..f05345c 100644 --- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java +++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java @@ -66,8 +66,8 @@ import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.MergePolicy; import org.apache.lucene.index.NoMergePolicy; -import org.apache.lucene.index.TieredMergePolicy; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.IOContext; @@ -399,6 +399,8 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate> { Analyzer analyzer = _fieldManager.getAnalyzerForIndex(); _conf = new IndexWriterConfig(LuceneVersionConstant.LUCENE_VERSION, analyzer); + MergePolicy mergePolicy = (MergePolicy) _conf.getMergePolicy(); + mergePolicy.setNoCFSRatio(0.0); _overFlowConf = new IndexWriterConfig(LuceneVersionConstant.LUCENE_VERSION, analyzer); _overFlowConf.setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES);
