Merge branch 'master' into 0.3.0-lucene-upgrade
Conflicts:
blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java
blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/9c7e5f38
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/9c7e5f38
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/9c7e5f38
Branch: refs/heads/0.3.0-lucene-upgrade
Commit: 9c7e5f389bd831b79859ea33261b2d3bfc4b08c3
Parents: a6009d6 292de85
Author: Aaron McCurry <[email protected]>
Authored: Sun Nov 3 10:19:01 2013 -0500
Committer: Aaron McCurry <[email protected]>
Committed: Sun Nov 3 10:19:01 2013 -0500
----------------------------------------------------------------------
.../blur/manager/AliasBlurFilterCache.java | 213 +++
.../apache/blur/manager/BlurFilterCache.java | 44 +-
.../blur/manager/DefaultBlurFilterCache.java | 29 +-
.../org/apache/blur/manager/IndexManager.java | 126 +-
.../apache/blur/manager/QueryParserUtil.java | 29 +-
.../clusterstatus/ZookeeperClusterStatus.java | 117 +-
.../indexserver/CloseableExecutorService.java | 4 +
.../indexserver/DistributedIndexServer.java | 91 +-
.../manager/indexserver/LocalIndexServer.java | 14 +-
.../MasterBasedDistributedLayoutFactory.java | 8 +-
.../blur/manager/results/LazyBlurResult.java | 4 +
.../blur/manager/writer/BlurIndexCloser.java | 7 +-
.../blur/manager/writer/BlurIndexReadOnly.java | 74 +
.../blur/manager/writer/BlurIndexReader.java | 2 +
.../blur/manager/writer/BlurIndexRefresher.java | 15 +-
.../blur/manager/writer/BlurNRTIndex.java | 128 +-
.../blur/manager/writer/IndexImporter.java | 30 +-
.../manager/writer/SharedMergeScheduler.java | 23 +-
.../manager/writer/TransactionRecorder.java | 14 +-
.../blur/thrift/BlurControllerServer.java | 126 +-
.../org/apache/blur/thrift/BlurShardServer.java | 20 +-
.../blur/thrift/ThriftBlurShardServer.java | 42 +-
.../org/apache/blur/utils/GCWatcherJdk7.java | 20 +-
.../blur/utils/TableShardCountCollapser.java | 8 +-
.../blur/manager/AliasBlurFilterCacheTest.java | 123 ++
.../apache/blur/manager/IndexManagerTest.java | 56 +-
.../ZookeeperClusterStatusTest.java | 14 +-
.../manager/writer/BlurIndexReaderTest.java | 12 +-
.../blur/manager/writer/BlurNRTIndexTest.java | 10 +-
.../writer/DirectoryReferenceCounterTest.java | 6 +-
.../org/apache/blur/thrift/BlurClusterTest.java | 51 +-
blur-gui/pom.xml | 19 +
.../org/apache/blur/mapreduce/BlurReducer.java | 3 +
.../blur/mapreduce/lib/BlurOutputFormat.java | 75 +-
.../blur/mapreduce/lib/CopyRateDirectory.java | 8 +-
.../apache/blur/analysis/BaseFieldManager.java | 37 +-
.../org/apache/blur/analysis/FieldManager.java | 12 +
.../blur/analysis/FieldTypeDefinition.java | 2 +
.../type/CustomFieldTypeDefinition.java | 71 +-
.../apache/blur/analysis/type/ExampleType.java | 110 ++
.../type/FieldLessFieldTypeDefinition.java | 5 +
.../type/NumericFieldTypeDefinition.java | 7 +-
.../type/StoredFieldTypeDefinition.java | 5 +
.../type/StringFieldTypeDefinition.java | 11 +-
.../analysis/type/TextFieldTypeDefinition.java | 5 +
...lPointVectorStrategyFieldTypeDefinition.java | 10 -
.../org/apache/blur/filter/FilterCache.java | 140 ++
.../org/apache/blur/filter/IndexFileBitSet.java | 123 ++
.../blur/filter/IndexFileBitSetIterator.java | 177 ++
.../apache/blur/filter/IndexInputLongArray.java | 40 +
.../blur/lucene/search/BlurQueryParser.java | 92 +-
.../blur/analysis/BaseFieldManagerTest.java | 51 +-
.../org/apache/blur/filter/FilterCacheTest.java | 131 ++
.../apache/blur/filter/IndexFileBitSetTest.java | 78 +
.../blur/lucene/search/SuperParserTest.java | 8 +
.../blur/shell/AddColumnDefinitionCommand.java | 6 +-
.../blur/shell/DiscoverFileBufferSizeUtil.java | 328 ++++
.../main/java/org/apache/blur/shell/Main.java | 1 +
.../apache/blur/lucene/codec/Blur021Codec.java | 125 ++
.../lucene/codec/Blur021LiveDocsFormat.java | 48 +
.../org/apache/blur/lucene/codec/Testing.java | 37 +
.../refcounter/DirectoryReferenceCounter.java | 129 +-
.../refcounter/DirectoryReferenceFileGC.java | 7 +-
.../store/refcounter/IndexInputCloser.java | 94 -
.../store/refcounter/IndexInputReference.java | 79 +
.../apache/blur/lucene/warmup/IndexWarmup.java | 5 +-
.../blur/store/BlockCacheDirectoryFactory.java | 4 +-
.../store/BlockCacheDirectoryFactoryV1.java | 5 +-
.../store/BlockCacheDirectoryFactoryV2.java | 118 +-
.../blur/store/blockcache_v2/BaseCache.java | 208 ++-
.../apache/blur/store/blockcache_v2/Cache.java | 3 +-
.../store/blockcache_v2/CacheDirectory.java | 40 +-
.../store/blockcache_v2/CacheIndexInput.java | 208 ++-
.../store/blockcache_v2/CacheIndexOutput.java | 3 +-
.../blur/store/blockcache_v2/CacheValue.java | 35 +
.../store/blockcache_v2/FileNameFilter.java | 2 +-
.../apache/blur/store/blockcache_v2/Quiet.java | 2 +-
.../apache/blur/store/blockcache_v2/Size.java | 2 +-
.../cachevalue/BaseCacheValue.java | 17 +
.../cachevalue/ByteArrayCacheValue.java | 12 +
.../cachevalue/UnsafeCacheValue.java | 32 +-
.../services/org.apache.lucene.codecs.Codec | 16 +
.../blur/store/BaseDirectoryTestSuite.java | 5 +-
.../blur/store/CacheDirectoryTestSuite.java | 27 +-
.../blur/store/HdfsDirectoryTestSuite.java | 5 +
.../store/SoftlinkHdfsDirectoryTestSuite.java | 5 +
.../store/blockcache_v2/CacheDirectoryTest.java | 21 +-
.../blockcache_v2/CacheIndexInputTest.java | 5 +
blur-thrift/pom.xml | 24 +
.../apache/blur/thrift/BlurClientManager.java | 2 +-
.../java/org/apache/blur/thrift/ClientPool.java | 125 +-
.../org/apache/blur/thrift/generated/Blur.java | 1737 +++++++++++++++---
blur-util/generate_docs.sh | 19 +
.../java/org/apache/blur/BlurConfiguration.java | 25 +-
.../blur/doc/CreateBlurServerSetupHtmlPage.java | 114 ++
.../apache/blur/metrics/MetricsConstants.java | 5 +
.../org/apache/blur/utils/BlurConstants.java | 23 +-
.../apache/blur/zookeeper/WatchChildren.java | 1 -
.../src/main/resources/blur-default.properties | 41 +-
.../src/main/scripts/interface/Blur.thrift | 11 +
.../main/scripts/interface/gen-html/Blur.html | 12 +
.../main/scripts/interface/gen-html/index.html | 1 +
.../org/apache/blur/thrift/generated/Blur.java | 1737 +++++++++++++++---
.../src/main/scripts/interface/gen-js/Blur.js | 621 +++++--
.../scripts/interface/gen-perl/Blur/Blur.pm | 545 ++++--
.../src/main/scripts/interface/gen-rb/blur.rb | 65 +
docs/Blur.html | 12 +
docs/cluster-setup.base.html | 354 ++++
docs/cluster-setup.html | 186 +-
docs/data-model.html | 137 ++
docs/using-blur.html | 16 +
pom.xml | 1 +
112 files changed, 8260 insertions(+), 1868 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/9c7e5f38/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
----------------------------------------------------------------------
diff --cc
blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
index a740686,f6912f2..fcdc414
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
@@@ -111,21 -112,20 +113,21 @@@ public class BlurNRTIndex extends BlurI
SnapshotDeletionPolicy sdp;
if (snapshotsDirectoryExists()) {
// load existing snapshots
- sdp = new SnapshotDeletionPolicy(_tableContext.getIndexDeletionPolicy(),
loadExistingSnapshots());
+ sdp = new
SnapshotDeletionPolicy(_tableContext.getIndexDeletionPolicy(),
loadExistingSnapshots());
} else {
sdp = new
SnapshotDeletionPolicy(_tableContext.getIndexDeletionPolicy());
- }
+ }
conf.setIndexDeletionPolicy(sdp);
+ conf.setMergedSegmentWarmer(new FieldBasedWarmer(shardContext,
_isClosed));
+ MergePolicy mergePolicy = (MergePolicy) conf.getMergePolicy();
+ mergePolicy.setNoCFSRatio(0.0);
-
+
- TieredMergePolicy mergePolicy = (TieredMergePolicy) conf.getMergePolicy();
- mergePolicy.setUseCompoundFile(false);
conf.setMergeScheduler(mergeScheduler.getMergeScheduler());
- DirectoryReferenceCounter referenceCounter = new
DirectoryReferenceCounter(directory, gc, closer);
+ DirectoryReferenceCounter referenceCounter = new
DirectoryReferenceCounter(directory, gc);
// This directory allows for warm up by adding tracing ability.
TraceableDirectory dir = new TraceableDirectory(referenceCounter);
-
+
SimpleTimer simpleTimer = new SimpleTimer();
simpleTimer.start("writerOpen");
_writer = new BlurIndexWriter(dir, conf, true);
@@@ -143,7 -143,7 +145,7 @@@
_trackingWriter = new TrackingIndexWriter(_writer);
_indexImporter = new IndexImporter(_trackingWriter, _lock, _shardContext,
TimeUnit.SECONDS, 10);
- _nrtManagerRef.set(new SearcherManager(_writer,
APPLY_ALL_DELETES,_searcherFactory));
- _nrtManagerRef.set(new NRTManager(_trackingWriter, _searcherFactory,
APPLY_ALL_DELETES));
++ _nrtManagerRef.set(new SearcherManager(_writer, APPLY_ALL_DELETES,
_searcherFactory));
// start commiter
_committer = new Thread(new Committer());
@@@ -153,7 -153,7 +155,8 @@@
// start refresher
double targetMinStaleSec = _tableContext.getTimeBetweenRefreshs() /
1000.0;
- _refresher = new
ControlledRealTimeReopenThread<IndexSearcher>(_trackingWriter,getNRTManager(),
targetMinStaleSec * 10, targetMinStaleSec);
- _refresher = new NRTManagerReopenThread(getNRTManager(),
targetMinStaleSec * 10, targetMinStaleSec);
++ _refresher = new
ControlledRealTimeReopenThread<IndexSearcher>(_trackingWriter, getNRTManager(),
++ targetMinStaleSec * 10, targetMinStaleSec);
_refresher.setName("Refresh Thread [" + _tableContext.getTable() + "/" +
shardContext.getShard() + "]");
_refresher.setDaemon(true);
_refresher.start();
@@@ -189,11 -188,10 +191,11 @@@
snapshots.put(snapshotName, segmentsFilename);
}
}
-
++
return snapshots;
}
-
- private boolean snapshotsDirectoryExists() throws IOException{
+
+ private boolean snapshotsDirectoryExists() throws IOException {
Path shardHdfsDirPath = _shardContext.getHdfsDirPath();
FileSystem fileSystem = getFileSystem();
Path shardSnapshotsDirPath = new Path(shardHdfsDirPath,
SNAPSHOTS_FOLDER_NAME);
@@@ -291,7 -289,7 +293,7 @@@
refresh();
}
if (waitToBeVisible && getNRTManager().getCurrentSearchingGen() <
generation) {
- _refresher.waitForGeneration(generation);
- getNRTManager().waitForGeneration(generation);
++ _refresher.waitForGeneration(generation);
}
}
@@@ -343,27 -341,23 +345,26 @@@
@Override
public void createSnapshot(String name) throws IOException {
SnapshotDeletionPolicy snapshotter = getSnapshotter();
- Map<String, String> existingSnapshots = snapshotter.getSnapshots();
- if (existingSnapshots.containsKey(name)) {
- LOG.error("A Snapshot already exists with the same name [{0}] on
[{1}/{2}].", name, _tableContext.getTable(),
- _shardContext.getShard());
- throw new IOException("A Snapshot already exists with the same name ["
+ name + "] on " + "["
- + _tableContext.getTable() + "/" + _shardContext.getShard() + "].");
+ List<IndexCommit> snapshots = snapshotter.getSnapshots();
-
- // Map<String, String> existingSnapshots = snapshotter.getSnapshots();
++
+ for (IndexCommit indexCommit : snapshots) {
- if (indexCommit.getSegmentsFileName().equals(name)) {
- LOG.error("A Snapshot already exists with the same name [{0}] on
[{1}/{2}].",
- name, _tableContext.getTable(), _shardContext.getShard());
- throw new IOException("A Snapshot already exists with the same
name [" + name + "] on " +
- "["+_tableContext.getTable()+"/"+_shardContext.getShard()+"].");
- }
++ if (indexCommit.getSegmentsFileName().equals(name)) {
++ LOG.error("A Snapshot already exists with the same name [{0}] on
[{1}/{2}].", name, _tableContext.getTable(),
++ _shardContext.getShard());
++ throw new IOException("A Snapshot already exists with the same name
[" + name + "] on " + "["
++ + _tableContext.getTable() + "/" + _shardContext.getShard() +
"].");
++ }
}
_writer.commit();
- IndexCommit indexCommit = snapshotter.snapshot(name);
+ IndexCommit indexCommit = snapshotter.snapshot();
-
+
/*
* Persist the snapshots info into a tmp file under the snapshots
sub-folder
- * and once writing is finished, close the writer.
- * Now rename the tmp file to an actual snapshots file.
- * This make the file write an atomic operation
+ * and once writing is finished, close the writer. Now rename the tmp file
+ * to an actual snapshots file. This make the file write an atomic
operation
*
- * The name of the file is the snapshot name and its contents specify the
segments file name
+ * The name of the file is the snapshot name and its contents specify the
+ * segments file name
*/
String segmentsFilename = indexCommit.getSegmentsFileName();
FileSystem fileSystem = getFileSystem();
http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/9c7e5f38/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/9c7e5f38/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/9c7e5f38/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/9c7e5f38/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java
----------------------------------------------------------------------
diff --cc blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java
index bf629bd,26d1122..fe7e312
--- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java
+++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java
@@@ -304,8 -305,9 +305,9 @@@ public class BlurReducer extends Reduce
if (optimize) {
context.setStatus("Starting Copy-Optimize Phase");
IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION,
_analyzer);
+ conf.setCodec(new Blur021Codec());
- TieredMergePolicy policy = (TieredMergePolicy) conf.getMergePolicy();
- policy.setUseCompoundFile(false);
+ MergePolicy mergePolicy = (MergePolicy) conf.getMergePolicy();
+ mergePolicy.setNoCFSRatio(0.0);
long s = System.currentTimeMillis();
IndexWriter writer = new IndexWriter(getBiggerBuffers(destDirectory),
conf);
writer.addIndexes(reader);
@@@ -459,10 -461,11 +461,11 @@@
nullCheck(_directory);
nullCheck(_analyzer);
IndexWriterConfig config = new IndexWriterConfig(LUCENE_VERSION,
_analyzer);
+ config.setCodec(new Blur021Codec());
+ MergePolicy mergePolicy = (MergePolicy) config.getMergePolicy();
+ mergePolicy.setNoCFSRatio(0.0);
config.setSimilarity(new FairSimilarity());
config.setRAMBufferSizeMB(_blurTask.getRamBufferSizeMB());
- TieredMergePolicy mergePolicy = (TieredMergePolicy)
config.getMergePolicy();
- mergePolicy.setUseCompoundFile(false);
_writer = new IndexWriter(_directory, config);
}
http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/9c7e5f38/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
----------------------------------------------------------------------
diff --cc
blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
index f05345c,caa3504..79e3ffc
---
a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
+++
b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
@@@ -399,8 -400,9 +400,9 @@@ public class BlurOutputFormat extends O
Analyzer analyzer = _fieldManager.getAnalyzerForIndex();
_conf = new IndexWriterConfig(LuceneVersionConstant.LUCENE_VERSION,
analyzer);
+ _conf.setCodec(new Blur021Codec());
- TieredMergePolicy mergePolicy = (TieredMergePolicy)
_conf.getMergePolicy();
- mergePolicy.setUseCompoundFile(false);
+ MergePolicy mergePolicy = (MergePolicy) _conf.getMergePolicy();
+ mergePolicy.setNoCFSRatio(0.0);
_overFlowConf = new
IndexWriterConfig(LuceneVersionConstant.LUCENE_VERSION, analyzer);
_overFlowConf.setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES);
http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/9c7e5f38/pom.xml
----------------------------------------------------------------------
