Updated Branches: refs/heads/0.3.0-lucene-upgrade [created] 7d8bc4220
BLUR-208: Upgrade blur to lucene 4.4 version Code is in broken state Signed-off-by: Aaron McCurry <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/7d8bc422 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/7d8bc422 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/7d8bc422 Branch: refs/heads/0.3.0-lucene-upgrade Commit: 7d8bc4220816a59c2b713c01a512c8d3ec56e309 Parents: fda400d Author: Saurabh Gupta <[email protected]> Authored: Sun Oct 13 07:37:22 2013 +0100 Committer: Aaron McCurry <[email protected]> Committed: Mon Oct 14 09:18:31 2013 -0400 ---------------------------------------------------------------------- .../blur/manager/writer/BlurNRTIndex.java | 43 +++++++++++--------- .../blur/manager/writer/IndexImporter.java | 2 +- .../manager/writer/TransactionRecorder.java | 2 +- .../blur/server/IndexSearcherClosableNRT.java | 6 +-- .../manager/writer/BlurIndexReaderTest.java | 2 +- .../blur/manager/writer/IndexImporterTest.java | 2 +- .../utils/TableShardCountCollapserTest.java | 4 +- .../org/apache/blur/mapreduce/BlurReducer.java | 4 -- .../blur/mapreduce/lib/BlurOutputFormat.java | 2 - .../blur/lucene/search/FairSimilarity.java | 9 ++++ .../blur/lucene/LuceneVersionConstant.java | 2 +- pom.xml | 2 +- 12 files changed, 42 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7d8bc422/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java index d593089..cd7df07 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java @@ -63,12 +63,12 @@ import org.apache.lucene.index.IndexDeletionPolicy; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.SnapshotDeletionPolicy; -import org.apache.lucene.index.TieredMergePolicy; +import org.apache.lucene.index.TrackingIndexWriter; +import org.apache.lucene.search.ControlledRealTimeReopenThread; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.NRTManager; -import org.apache.lucene.search.NRTManager.TrackingIndexWriter; -import org.apache.lucene.search.NRTManagerReopenThread; +import org.apache.lucene.search.ReferenceManager; import org.apache.lucene.search.SearcherFactory; +import org.apache.lucene.search.SearcherManager; import org.apache.lucene.store.Directory; public class BlurNRTIndex extends BlurIndex { @@ -78,13 +78,13 @@ public class BlurNRTIndex extends BlurIndex { private static final String SNAPSHOTS_FOLDER_NAME = "snapshots"; private static final String SNAPSHOTS_TMPFILE_EXTENSION = ".tmp"; - private final AtomicReference<NRTManager> _nrtManagerRef = new AtomicReference<NRTManager>(); + private final AtomicReference<SearcherManager> _nrtManagerRef = new AtomicReference<SearcherManager>(); private final AtomicBoolean _isClosed = new AtomicBoolean(); private final BlurIndexWriter _writer; private final Thread _committer; private final SearcherFactory _searcherFactory; private final Directory _directory; - private final NRTManagerReopenThread _refresher; + private final ControlledRealTimeReopenThread<IndexSearcher> _refresher; private final TableContext _tableContext; private final ShardContext _shardContext; private final TransactionRecorder _recorder; @@ -110,15 +110,13 @@ public class BlurNRTIndex extends BlurIndex { SnapshotDeletionPolicy sdp; if (snapshotsDirectoryExists()) { // load existing snapshots - sdp = new SnapshotDeletionPolicy(_tableContext.getIndexDeletionPolicy(), loadExistingSnapshots()); + sdp = new SnapshotDeletionPolicy(_tableContext.getIndexDeletionPolicy(), loadExistingSnapshots()); } else { sdp = new SnapshotDeletionPolicy(_tableContext.getIndexDeletionPolicy()); - } + } conf.setIndexDeletionPolicy(sdp); conf.setMergedSegmentWarmer(new FieldBasedWarmer(shardContext, _isClosed)); - TieredMergePolicy mergePolicy = (TieredMergePolicy) conf.getMergePolicy(); - mergePolicy.setUseCompoundFile(false); conf.setMergeScheduler(mergeScheduler.getMergeScheduler()); DirectoryReferenceCounter referenceCounter = new DirectoryReferenceCounter(directory, gc, closer); @@ -142,7 +140,7 @@ public class BlurNRTIndex extends BlurIndex { _trackingWriter = new TrackingIndexWriter(_writer); _indexImporter = new IndexImporter(_trackingWriter, _lock, _shardContext, TimeUnit.SECONDS, 10); - _nrtManagerRef.set(new NRTManager(_trackingWriter, _searcherFactory, APPLY_ALL_DELETES)); + _nrtManagerRef.set(new SearcherManager(_writer, APPLY_ALL_DELETES,_searcherFactory)); // start commiter _committer = new Thread(new Committer()); @@ -152,7 +150,7 @@ public class BlurNRTIndex extends BlurIndex { // start refresher double targetMinStaleSec = _tableContext.getTimeBetweenRefreshs() / 1000.0; - _refresher = new NRTManagerReopenThread(getNRTManager(), targetMinStaleSec * 10, targetMinStaleSec); + _refresher = new ControlledRealTimeReopenThread<IndexSearcher>(_trackingWriter,getNRTManager(), targetMinStaleSec * 10, targetMinStaleSec); _refresher.setName("Refresh Thread [" + _tableContext.getTable() + "/" + shardContext.getShard() + "]"); _refresher.setDaemon(true); _refresher.start(); @@ -187,6 +185,7 @@ public class BlurNRTIndex extends BlurIndex { snapshots.put(snapshotName, segmentsFilename); } } + return snapshots; } @@ -245,7 +244,7 @@ public class BlurNRTIndex extends BlurIndex { return indexSearcherClosable; } - private NRTManager getNRTManager() { + private ReferenceManager<IndexSearcher> getNRTManager() { return _nrtManagerRef.get(); } @@ -288,7 +287,7 @@ public class BlurNRTIndex extends BlurIndex { refresh(); } if (waitToBeVisible && getNRTManager().getCurrentSearchingGen() < generation) { - getNRTManager().waitForGeneration(generation); + _refresher.waitForGeneration(generation); } } @@ -340,12 +339,16 @@ public class BlurNRTIndex extends BlurIndex { @Override public void createSnapshot(String name) throws IOException { SnapshotDeletionPolicy snapshotter = getSnapshotter(); - Map<String, String> existingSnapshots = snapshotter.getSnapshots(); - if (existingSnapshots.containsKey(name)) { - LOG.error("A Snapshot already exists with the same name [{0}] on [{1}/{2}].", - name, _tableContext.getTable(), _shardContext.getShard()); - throw new IOException("A Snapshot already exists with the same name [" + name + "] on " + - "["+_tableContext.getTable()+"/"+_shardContext.getShard()+"]."); + List<IndexCommit> snapshots = snapshotter.getSnapshots(); + +// Map<String, String> existingSnapshots = snapshotter.getSnapshots(); + for (IndexCommit indexCommit : snapshots) { + if (indexCommit.getSegmentsFileName().equals(name)) { + LOG.error("A Snapshot already exists with the same name [{0}] on [{1}/{2}].", + name, _tableContext.getTable(), _shardContext.getShard()); + throw new IOException("A Snapshot already exists with the same name [" + name + "] on " + + "["+_tableContext.getTable()+"/"+_shardContext.getShard()+"]."); + } } _writer.commit(); IndexCommit indexCommit = snapshotter.snapshot(name); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7d8bc422/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java b/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java index cbf5e91..34b6727 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java @@ -49,7 +49,7 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.search.NRTManager.TrackingIndexWriter; +import org.apache.lucene.index.TrackingIndexWriter; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7d8bc422/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java b/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java index 605fe32..6a0bf31 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java @@ -58,7 +58,7 @@ import org.apache.lucene.document.TextField; import org.apache.lucene.index.BlurIndexWriter; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.Term; -import org.apache.lucene.search.NRTManager.TrackingIndexWriter; +import org.apache.lucene.index.TrackingIndexWriter; public class TransactionRecorder extends TimerTask implements Closeable { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7d8bc422/blur-core/src/main/java/org/apache/blur/server/IndexSearcherClosableNRT.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/server/IndexSearcherClosableNRT.java b/blur-core/src/main/java/org/apache/blur/server/IndexSearcherClosableNRT.java index ddd28ab..b6b2fe7 100644 --- a/blur-core/src/main/java/org/apache/blur/server/IndexSearcherClosableNRT.java +++ b/blur-core/src/main/java/org/apache/blur/server/IndexSearcherClosableNRT.java @@ -21,15 +21,15 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicReference; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.search.NRTManager; +import org.apache.lucene.search.SearcherManager; import org.apache.lucene.store.Directory; public class IndexSearcherClosableNRT extends IndexSearcherClosable { - private final AtomicReference<NRTManager> _nrtManagerRef; + private final AtomicReference<SearcherManager> _nrtManagerRef; private final Directory _directory; - public IndexSearcherClosableNRT(IndexReader r, ExecutorService executor, AtomicReference<NRTManager> nrtManagerRef, + public IndexSearcherClosableNRT(IndexReader r, ExecutorService executor, AtomicReference<SearcherManager> nrtManagerRef, Directory directory) { super(r, executor); _nrtManagerRef = nrtManagerRef; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7d8bc422/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java index 59316f3..f98f9d7 100644 --- a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java +++ b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java @@ -136,7 +136,7 @@ public class BlurIndexReaderTest { } private void doWrite() throws CorruptIndexException, LockObtainFailedException, IOException { - IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer()); + IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_44, new KeywordAnalyzer()); BlurIndexWriter writer = new BlurIndexWriter(directory, conf); writer.addDocument(getDoc()); writer.close(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7d8bc422/blur-core/src/test/java/org/apache/blur/manager/writer/IndexImporterTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/manager/writer/IndexImporterTest.java b/blur-core/src/test/java/org/apache/blur/manager/writer/IndexImporterTest.java index e7724a5..cabf709 100644 --- a/blur-core/src/test/java/org/apache/blur/manager/writer/IndexImporterTest.java +++ b/blur-core/src/test/java/org/apache/blur/manager/writer/IndexImporterTest.java @@ -43,7 +43,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.search.NRTManager.TrackingIndexWriter; +import org.apache.lucene.index.TrackingIndexWriter; import org.apache.lucene.store.Directory; import org.junit.After; import org.junit.Before; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7d8bc422/blur-core/src/test/java/org/apache/blur/utils/TableShardCountCollapserTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/utils/TableShardCountCollapserTest.java b/blur-core/src/test/java/org/apache/blur/utils/TableShardCountCollapserTest.java index 28bf718..5be4457 100644 --- a/blur-core/src/test/java/org/apache/blur/utils/TableShardCountCollapserTest.java +++ b/blur-core/src/test/java/org/apache/blur/utils/TableShardCountCollapserTest.java @@ -94,9 +94,7 @@ public class TableShardCountCollapserTest { private static void createShard(Configuration configuration, int i, Path path, int totalShardCount) throws IOException { HdfsDirectory hdfsDirectory = new HdfsDirectory(configuration, path); - IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer()); - TieredMergePolicy mergePolicy = (TieredMergePolicy) conf.getMergePolicy(); - mergePolicy.setUseCompoundFile(false); + IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_44, new KeywordAnalyzer()); IndexWriter indexWriter = new IndexWriter(hdfsDirectory, conf); Partitioner<IntWritable, IntWritable> partitioner = new HashPartitioner<IntWritable, IntWritable>(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7d8bc422/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java ---------------------------------------------------------------------- diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java b/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java index 3669232..2e5773b 100644 --- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java +++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java @@ -304,8 +304,6 @@ public class BlurReducer extends Reducer<Text, BlurMutate, Text, BlurMutate> { if (optimize) { context.setStatus("Starting Copy-Optimize Phase"); IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, _analyzer); - TieredMergePolicy policy = (TieredMergePolicy) conf.getMergePolicy(); - policy.setUseCompoundFile(false); long s = System.currentTimeMillis(); IndexWriter writer = new IndexWriter(getBiggerBuffers(destDirectory), conf); writer.addIndexes(reader); @@ -461,8 +459,6 @@ public class BlurReducer extends Reducer<Text, BlurMutate, Text, BlurMutate> { IndexWriterConfig config = new IndexWriterConfig(LUCENE_VERSION, _analyzer); config.setSimilarity(new FairSimilarity()); config.setRAMBufferSizeMB(_blurTask.getRamBufferSizeMB()); - TieredMergePolicy mergePolicy = (TieredMergePolicy) config.getMergePolicy(); - mergePolicy.setUseCompoundFile(false); _writer = new IndexWriter(_directory, config); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7d8bc422/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java ---------------------------------------------------------------------- diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java index 2f59a20..725be30 100644 --- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java +++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java @@ -399,8 +399,6 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate> { Analyzer analyzer = _fieldManager.getAnalyzerForIndex(); _conf = new IndexWriterConfig(LuceneVersionConstant.LUCENE_VERSION, analyzer); - TieredMergePolicy mergePolicy = (TieredMergePolicy) _conf.getMergePolicy(); - mergePolicy.setUseCompoundFile(false); _overFlowConf = new IndexWriterConfig(LuceneVersionConstant.LUCENE_VERSION, analyzer); _overFlowConf.setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7d8bc422/blur-query/src/main/java/org/apache/blur/lucene/search/FairSimilarity.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/lucene/search/FairSimilarity.java b/blur-query/src/main/java/org/apache/blur/lucene/search/FairSimilarity.java index 8c8b945..c3e0277 100644 --- a/blur-query/src/main/java/org/apache/blur/lucene/search/FairSimilarity.java +++ b/blur-query/src/main/java/org/apache/blur/lucene/search/FairSimilarity.java @@ -58,4 +58,13 @@ public class FairSimilarity extends TFIDFSimilarity { // return 0; } + @Override + public float decodeNormValue(long norm) { + return 1; + } + + @Override + public long encodeNormValue(float f) { + return 1; + } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7d8bc422/blur-store/src/main/java/org/apache/blur/lucene/LuceneVersionConstant.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/lucene/LuceneVersionConstant.java b/blur-store/src/main/java/org/apache/blur/lucene/LuceneVersionConstant.java index 20ae366..ab8cf85 100644 --- a/blur-store/src/main/java/org/apache/blur/lucene/LuceneVersionConstant.java +++ b/blur-store/src/main/java/org/apache/blur/lucene/LuceneVersionConstant.java @@ -23,6 +23,6 @@ public class LuceneVersionConstant { /** * All Lucene Version references should refer to this constant. */ - public static final Version LUCENE_VERSION = Version.LUCENE_43; + public static final Version LUCENE_VERSION = Version.LUCENE_44; } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7d8bc422/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 05e03b6..4d6c831 100644 --- a/pom.xml +++ b/pom.xml @@ -175,7 +175,7 @@ under the License. <zookeeper.version>3.4.5</zookeeper.version> <log4j.version>1.2.15</log4j.version> <jersey.version>1.14</jersey.version> - <lucene.version>4.3.0</lucene.version> + <lucene.version>4.4.0</lucene.version> <junit.version>4.7</junit.version> <slf4j.version>1.6.1</slf4j.version> <commons-cli.version>1.2</commons-cli.version>
