http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e9dce540/blur-store/src/main/java/org/apache/blur/lucene/warmup/ThrottledIndexInput.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/lucene/warmup/ThrottledIndexInput.java b/blur-store/src/main/java/org/apache/blur/lucene/warmup/ThrottledIndexInput.java deleted file mode 100644 index 0b6d612..0000000 --- a/blur-store/src/main/java/org/apache/blur/lucene/warmup/ThrottledIndexInput.java +++ /dev/null @@ -1,164 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.blur.lucene.warmup; - -import java.io.IOException; - -import org.apache.lucene.store.IndexInput; - -/** - * This class has been derived from the - * org.apache.hadoop.tools.util.ThrottledInputStream class found in Hadoop. - * - * The ThrottledIndexInput provides bandwidth throttling on a specified - * IndexInput. It is implemented as a wrapper on top of another IndexInput - * instance. The throttling works by examining the number of bytes read from the - * underlying IndexInput from the beginning, and sleep()ing for a time interval - * if the byte-transfer is found exceed the specified tolerable maximum. (Thus, - * while the read-rate might exceed the maximum for a given short interval, the - * average tends towards the specified maximum, overall.) - */ -public class ThrottledIndexInput extends IndexInput { - - private static final long SLEEP_DURATION_MS = 1; - - private final IndexInput _rawStream; - private final double _maxBytesPerSec; - private final long _startTime; - - private long _bytesRead = 0; - private long _totalSleepTime = 0; - - public ThrottledIndexInput(IndexInput rawStream, long maxBytesPerSec) { - super("ThrottledIndexInput(" + rawStream.toString() + ")"); - _rawStream = rawStream; - _maxBytesPerSec = maxBytesPerSec; - _startTime = System.nanoTime(); - } - - public ThrottledIndexInput(IndexInput rawStream, long maxBytesPerSec, long bytesRead, long totalSleepTime, - long startTime) { - super("ThrottledIndexInput(" + rawStream.toString() + ")"); - _rawStream = rawStream; - _maxBytesPerSec = maxBytesPerSec; - _bytesRead = bytesRead; - _totalSleepTime = totalSleepTime; - _startTime = startTime; - } - - /** @inheritDoc */ - @Override - public long getFilePointer() { - return _rawStream.getFilePointer(); - } - - /** @inheritDoc */ - @Override - public void seek(long pos) throws IOException { - _rawStream.seek(pos); - } - - /** @inheritDoc */ - @Override - public long length() { - return _rawStream.length(); - } - - /** @inheritDoc */ - @Override - public byte readByte() throws IOException { - throttle(); - try { - return _rawStream.readByte(); - } finally { - _bytesRead++; - } - } - - /** @inheritDoc */ - @Override - public void readBytes(byte[] b, int offset, int len) throws IOException { - throttle(); - try { - _rawStream.readBytes(b, offset, len); - } finally { - _bytesRead += len; - } - } - - /** @inheritDoc */ - @Override - public void close() throws IOException { - _rawStream.close(); - } - - private void throttle() throws IOException { - while (getBytesPerSec() > _maxBytesPerSec) { - try { - Thread.sleep(SLEEP_DURATION_MS); - _totalSleepTime += SLEEP_DURATION_MS; - } catch (InterruptedException e) { - throw new IOException("Thread aborted", e); - } - } - } - - /** - * Getter for the number of bytes read from this stream, since creation. - * - * @return The number of bytes. - */ - public long getTotalBytesRead() { - return _bytesRead; - } - - /** - * Getter for the read-rate from this stream, since creation. Calculated as - * bytesRead/elapsedTimeSinceStart. - * - * @return Read rate, in bytes/sec. - */ - public double getBytesPerSec() { - double elapsed = (System.nanoTime() - _startTime) / 1000000000; - if (elapsed == 0) { - return _bytesRead; - } else { - return _bytesRead / elapsed; - } - } - - /** - * Getter the total time spent in sleep. - * - * @return Number of milliseconds spent in sleep. - */ - public long getTotalSleepTime() { - return _totalSleepTime; - } - - /** @inheritDoc */ - @Override - public String toString() { - return "ThrottledIndexInput{" + "bytesRead=" + _bytesRead + ", maxBytesPerSec=" + _maxBytesPerSec - + ", bytesPerSec=" + getBytesPerSec() + ", totalSleepTime=" + _totalSleepTime + '}'; - } - - public long getStartTime() { - return _startTime; - } - -} \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e9dce540/blur-store/src/main/java/org/apache/blur/lucene/warmup/TraceableDirectory.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/lucene/warmup/TraceableDirectory.java b/blur-store/src/main/java/org/apache/blur/lucene/warmup/TraceableDirectory.java deleted file mode 100644 index 7e092c3..0000000 --- a/blur-store/src/main/java/org/apache/blur/lucene/warmup/TraceableDirectory.java +++ /dev/null @@ -1,152 +0,0 @@ -package org.apache.blur.lucene.warmup; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import java.io.IOException; -import java.util.Collection; - -import org.apache.blur.store.hdfs.DirectoryDecorator; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.Lock; -import org.apache.lucene.store.LockFactory; - -public class TraceableDirectory extends Directory implements DirectoryDecorator { - - private final Directory _dir; - /** - * One might feel compelled to make this field volatile because of the - * massively parallel environment that Blur operates. Please do NOT! This - * field cannot be volatile for performance reasons. For every query in Lucene - * there are a number of clones made of each file handle in each segment in - * each index. If this were volatile every thread would have to check main - * memory for this field every time a clone was called (see - * {@link TraceableIndexInput}). In fact I want this field to the remain false - * in all cases, except for when a warm up trace is needed. If that occurs the - * Thread running the actual the trace will set this field to true for a brief - * moment while all the clones are made and the trace is performed (the clones - * are also made with this this trace thread), this will likely just alter the - * value for that thread and not commit it back to main memory (heap). After - * the trace is complete is will return the field to false. During the time - * when the field is true, if any other field actually reads the value as true - * there is a second {@link ThreadLocal} field in {@link IndexWarmup} that - * prevents any other threads from running a trace. However there will be a - * small performance penalty while that situation occurs, because accessing a - * {@link ThreadLocal} field is fairly expensive. - * - * In short please leave this the way it is, if you really want to change it - * please post a question on the mail list first. - */ - private boolean _trace = false; - private ThreadLocal<IndexTracer> _indexTracer = new ThreadLocal<IndexTracer>(); - - public TraceableDirectory(Directory dir) { - _dir = dir; - } - - public void trace(String name, long filePointer) { - if (_trace) { - if (_indexTracer != null) { - _indexTracer.get().trace(name, filePointer); - } - } - } - - public IndexInput openInput(String name, IOContext context) throws IOException { - return new TraceableIndexInput(this, name, context, _dir.openInput(name, context)); - } - - public String[] listAll() throws IOException { - return _dir.listAll(); - } - - public boolean fileExists(String name) throws IOException { - return _dir.fileExists(name); - } - - public void deleteFile(String name) throws IOException { - _dir.deleteFile(name); - } - - public long fileLength(String name) throws IOException { - return _dir.fileLength(name); - } - - public IndexOutput createOutput(String name, IOContext context) throws IOException { - return _dir.createOutput(name, context); - } - - public void sync(Collection<String> names) throws IOException { - _dir.sync(names); - } - - public void close() throws IOException { - _dir.close(); - } - - public boolean isTrace() { - return _trace; - } - - public void setTrace(boolean trace) { - _trace = trace; - } - - public void setIndexTracer(IndexTracer indexTracer) { - _indexTracer.set(indexTracer); - } - - public Lock makeLock(String name) { - return _dir.makeLock(name); - } - - public void clearLock(String name) throws IOException { - _dir.clearLock(name); - } - - public void setLockFactory(LockFactory lockFactory) throws IOException { - _dir.setLockFactory(lockFactory); - } - - public LockFactory getLockFactory() { - return _dir.getLockFactory(); - } - - public String getLockID() { - return _dir.getLockID(); - } - - public String toString() { - return _dir.toString(); - } - - public void copy(Directory to, String src, String dest, IOContext context) throws IOException { - _dir.copy(to, src, dest, context); - } - - public IndexInputSlicer createSlicer(String name, IOContext context) throws IOException { - return _dir.createSlicer(name, context); - } - - @Override - public Directory getOriginalDirectory() { - return _dir; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e9dce540/blur-store/src/main/java/org/apache/blur/lucene/warmup/TraceableIndexInput.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/lucene/warmup/TraceableIndexInput.java b/blur-store/src/main/java/org/apache/blur/lucene/warmup/TraceableIndexInput.java deleted file mode 100644 index 9ad3cda..0000000 --- a/blur-store/src/main/java/org/apache/blur/lucene/warmup/TraceableIndexInput.java +++ /dev/null @@ -1,93 +0,0 @@ -package org.apache.blur.lucene.warmup; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import java.io.IOException; - -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; - -public class TraceableIndexInput extends IndexInput { - - private final TraceableDirectory _traceableDirectory; - private final String _name; - private IndexInput _input; - private boolean _traceOn; - - public TraceableIndexInput(TraceableDirectory traceableDirectory, String name, IOContext context, IndexInput input) { - super("TraceableIndexInput(" + input.toString() + ")"); - _traceableDirectory = traceableDirectory; - _input = input; - _traceOn = _traceableDirectory.isTrace(); - _name = name; - } - - @Override - public void close() throws IOException { - _input.close(); - } - - @Override - public long getFilePointer() { - return _input.getFilePointer(); - } - - @Override - public void seek(long pos) throws IOException { - _input.seek(pos); - } - - @Override - public long length() { - return _input.length(); - } - - @Override - public byte readByte() throws IOException { - trace(); - return _input.readByte(); - } - - private void trace() { - if (_traceOn) { - if (IndexWarmup.isRunTrace()) { - _traceableDirectory.trace(_name, _input.getFilePointer()); - } - } - } - - @Override - public void readBytes(byte[] b, int offset, int len) throws IOException { - trace(); - _input.readBytes(b, offset, len); - } - - @Override - public void readBytes(byte[] b, int offset, int len, boolean useBuffer) throws IOException { - trace(); - _input.readBytes(b, offset, len, useBuffer); - } - - @Override - public IndexInput clone() { - TraceableIndexInput clone = (TraceableIndexInput) super.clone(); - clone._input = _input.clone(); - clone._traceOn = _traceableDirectory.isTrace(); - return clone; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e9dce540/blur-store/src/test/java/org/apache/blur/lucene/warmup/IndexWarmupTest.java ---------------------------------------------------------------------- diff --git a/blur-store/src/test/java/org/apache/blur/lucene/warmup/IndexWarmupTest.java b/blur-store/src/test/java/org/apache/blur/lucene/warmup/IndexWarmupTest.java deleted file mode 100644 index 2fcc969..0000000 --- a/blur-store/src/test/java/org/apache/blur/lucene/warmup/IndexWarmupTest.java +++ /dev/null @@ -1,183 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.blur.lucene.warmup; - -import java.io.File; -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Random; -import java.util.concurrent.atomic.AtomicBoolean; - -import org.apache.lucene.analysis.standard.StandardAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field.Store; -import org.apache.lucene.document.TextField; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.IndexableField; -import org.apache.lucene.index.TieredMergePolicy; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.util.OpenBitSet; -import org.apache.lucene.util.Version; -import org.junit.Test; - -public class IndexWarmupTest { - - private Random _random = new Random(); - private int _numberOfFields = 10; - - @Test - public void testIndexWarmup() throws IOException { - File file = new File("./target/tmp/indexwarmup-test"); - Directory dir = FSDirectory.open(file); - - Directory directory = new TraceableDirectory(new SlowAccessDirectory(dir)); - IndexReader indexReader = getIndexReader(directory); - int maxSampleSize = 1000; - int blockSize = 8192; - long totalLookups = 0; - for (String s : directory.listAll()) { - if (s.endsWith(".pos") || s.endsWith(".doc") || s.endsWith(".tim")) { - long fileLength = directory.fileLength(s); - long maxHits = (long) Math.ceil(fileLength / (double) blockSize); - totalLookups += maxHits; - System.out.println("file [" + s + "] size [" + fileLength + "] maxhits [" + maxHits + "]"); - } - } - AtomicBoolean stop = new AtomicBoolean(); - AtomicBoolean isClosed = new AtomicBoolean(); - SlowAccessDirectory._reads.set(0); - IndexWarmup indexWarmup = new IndexWarmup(isClosed, stop, maxSampleSize, Long.MAX_VALUE); - long t1 = System.nanoTime(); - Map<String, List<IndexTracerResult>> sampleIndex = indexWarmup.sampleIndex(indexReader, "test"); - long sampleReads = SlowAccessDirectory._reads.get(); - SlowAccessDirectory._reads.set(0); - long t2 = System.nanoTime(); - for (int i = 0; i < _numberOfFields; i++) { - indexWarmup.warmFile(indexReader, sampleIndex, "test" + i, "test"); - } - long t3 = System.nanoTime(); - System.out.println((t2 - t1) / 1000000.0 + " " + (t3 - t2) / 1000000.0); - System.out.println(totalLookups + " " + sampleReads + " " + SlowAccessDirectory._reads.get()); - } - - @Test - public void testIndexWarmupBitSet() throws IOException { - File file = new File("./target/tmp/indexwarmup-test"); - Directory dir = FSDirectory.open(file); - - Directory directory = new TraceableDirectory(new SlowAccessDirectory(dir)); - IndexReader indexReader = getIndexReader(directory); - int maxSampleSize = 1000; - int blockSize = 8192; - // int blockSize = 1024 * 1024; - long totalLookups = 0; - for (String s : directory.listAll()) { - if (s.endsWith(".pos") || s.endsWith(".doc") || s.endsWith(".tim")) { - long fileLength = directory.fileLength(s); - long maxHits = (long) Math.ceil(fileLength / (double) blockSize); - totalLookups += maxHits; - System.out.println("file [" + s + "] size [" + fileLength + "] maxhits [" + maxHits + "]"); - } - } - AtomicBoolean stop = new AtomicBoolean(); - AtomicBoolean isClosed = new AtomicBoolean(); - SlowAccessDirectory._reads.set(0); - IndexWarmup indexWarmup = new IndexWarmup(isClosed, stop, maxSampleSize, Long.MAX_VALUE); - long t1 = System.nanoTime(); - Map<String, List<IndexTracerResult>> sampleIndex = indexWarmup.sampleIndex(indexReader, "test"); - long sampleReads = SlowAccessDirectory._reads.get(); - SlowAccessDirectory._reads.set(0); - long t2 = System.nanoTime(); - Map<String, OpenBitSet> filePartsToWarm = new HashMap<String, OpenBitSet>(); - for (int i = 0; i < _numberOfFields; i++) { - indexWarmup.getFilePositionsToWarm(indexReader, sampleIndex, "test" + i, "test", filePartsToWarm, blockSize); - } - indexWarmup.warmFile(indexReader, filePartsToWarm, "test", blockSize, 1024 * 1024); - long t3 = System.nanoTime(); - - for (Entry<String, OpenBitSet> e : filePartsToWarm.entrySet()) { - OpenBitSet bitSet = e.getValue(); - System.out.println(bitSet.length() + " " + bitSet.cardinality()); - } - - System.out.println((t2 - t1) / 1000000.0 + " " + (t3 - t2) / 1000000.0); - System.out.println(totalLookups + " " + sampleReads + " " + SlowAccessDirectory._reads.get()); - } - - private IndexReader getIndexReader(Directory directory) throws IOException { - if (!DirectoryReader.indexExists(directory)) { - long t1 = System.nanoTime(); - populate(directory); - long t2 = System.nanoTime(); - System.out.println((t2 - t1) / 1000000.0); - } - return DirectoryReader.open(directory); - } - - private void populate(Directory directory) throws IOException { - IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new StandardAnalyzer(Version.LUCENE_43)); - TieredMergePolicy mergePolicy = (TieredMergePolicy) conf.getMergePolicy(); - mergePolicy.setUseCompoundFile(false); - IndexWriter writer = new IndexWriter(directory, conf); - addDocs(writer); - writer.close(); - } - - private void addDocs(IndexWriter writer) throws IOException { - for (int i = 0; i < 20000; i++) { - writer.addDocument(getDoc()); - } - } - - private Iterable<? extends IndexableField> getDoc() { - Document document = new Document(); - document.add(new TextField(getFieldName(), getText(), Store.YES)); - return document; - } - - private String getText() { - StringBuilder builder = new StringBuilder(); - for (int i = 0; i < 1000; i++) { - builder.append(getWord()).append(' '); - } - return builder.toString(); - } - - private String getWord() { - StringBuilder builder = new StringBuilder(); - for (int i = 0; i < 10; i++) { - builder.append(getChar()); - } - return builder.toString(); - } - - private char getChar() { - return (char) (_random.nextInt(26) + 'a'); - } - - private String getFieldName() { - return "test" + _random.nextInt(_numberOfFields); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e9dce540/blur-store/src/test/java/org/apache/blur/lucene/warmup/SlowAccessDirectory.java ---------------------------------------------------------------------- diff --git a/blur-store/src/test/java/org/apache/blur/lucene/warmup/SlowAccessDirectory.java b/blur-store/src/test/java/org/apache/blur/lucene/warmup/SlowAccessDirectory.java deleted file mode 100644 index f9aff08..0000000 --- a/blur-store/src/test/java/org/apache/blur/lucene/warmup/SlowAccessDirectory.java +++ /dev/null @@ -1,149 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.blur.lucene.warmup; - -import java.io.IOException; -import java.util.Collection; -import java.util.concurrent.atomic.AtomicLong; - -import org.apache.lucene.store.BufferedIndexInput; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.Lock; -import org.apache.lucene.store.LockFactory; - -public class SlowAccessDirectory extends Directory { - private final Directory _directory; - - public static AtomicLong _reads = new AtomicLong(); - - static class SlowAccessIndexInput extends BufferedIndexInput { - - private IndexInput _indexInput; - - protected SlowAccessIndexInput(IndexInput indexInput) { - super(indexInput.toString(), 8192); - _indexInput = indexInput; - } - - private void delay() throws IOException { - synchronized (this) { - try { - wait(1); - } catch (InterruptedException e) { - throw new IOException(e); - } - } - } - - public void close() throws IOException { - _indexInput.close(); - } - - public long length() { - return _indexInput.length(); - } - - @Override - public SlowAccessIndexInput clone() { - SlowAccessIndexInput clone = (SlowAccessIndexInput) super.clone(); - clone._indexInput = _indexInput.clone(); - return clone; - } - - @Override - protected void readInternal(byte[] b, int offset, int length) throws IOException { - delay(); - _reads.incrementAndGet(); - long filePointer = getFilePointer(); - _indexInput.seek(filePointer); - _indexInput.readBytes(b, offset, length); - } - - @Override - protected void seekInternal(long pos) throws IOException { - - } - - } - - public SlowAccessDirectory(Directory directory) { - _directory = directory; - } - - public String[] listAll() throws IOException { - return _directory.listAll(); - } - - public boolean fileExists(String name) throws IOException { - return _directory.fileExists(name); - } - - public void deleteFile(String name) throws IOException { - _directory.deleteFile(name); - } - - public long fileLength(String name) throws IOException { - return _directory.fileLength(name); - } - - public IndexOutput createOutput(String name, IOContext context) throws IOException { - return _directory.createOutput(name, context); - } - - public void sync(Collection<String> names) throws IOException { - _directory.sync(names); - } - - public IndexInput openInput(String name, IOContext context) throws IOException { - return new SlowAccessIndexInput(_directory.openInput(name, context)); - } - - public Lock makeLock(String name) { - return _directory.makeLock(name); - } - - public void clearLock(String name) throws IOException { - _directory.clearLock(name); - } - - public void close() throws IOException { - _directory.close(); - } - - public void setLockFactory(LockFactory lockFactory) throws IOException { - _directory.setLockFactory(lockFactory); - } - - public LockFactory getLockFactory() { - return _directory.getLockFactory(); - } - - public String getLockID() { - return _directory.getLockID(); - } - - public void copy(Directory to, String src, String dest, IOContext context) throws IOException { - _directory.copy(to, src, dest, context); - } - - public IndexInputSlicer createSlicer(String name, IOContext context) throws IOException { - return _directory.createSlicer(name, context); - } -} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e9dce540/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java ---------------------------------------------------------------------- diff --git a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java index 97c5cda..43c0b83 100644 --- a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java +++ b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java @@ -78,17 +78,15 @@ public class BlurConstants { public static final String BLUR_MAX_HEAP_PER_ROW_FETCH = "blur.max.heap.per.row.fetch"; public static final String BLUR_MAX_RECORDS_PER_ROW_FETCH_REQUEST = "blur.max.records.per.row.fetch.request"; public static final String BLUR_SHARD_READ_INTERCEPTOR = "blur.shard.read.interceptor"; + public static final String BLUR_SHARD_INTERNAL_SEARCH_THREAD_COUNT = "blur.shard.internal.search.thread.count"; public static final String BLUR_SHARD_SERVER_THRIFT_THREAD_COUNT = "blur.shard.server.thrift.thread.count"; public static final String BLUR_SHARD_CACHE_MAX_TIMETOLIVE = "blur.shard.cache.max.timetolive"; public static final String BLUR_SHARD_FILTER_CACHE_CLASS = "blur.shard.filter.cache.class"; - public static final String BLUR_SHARD_INDEX_WARMUP_CLASS = "blur.shard.index.warmup.class"; public static final String BLUR_INDEXMANAGER_SEARCH_THREAD_COUNT = "blur.indexmanager.search.thread.count"; public static final String BLUR_INDEXMANAGER_MUTATE_THREAD_COUNT = "blur.indexmanager.mutate.thread.count"; public static final String BLUR_INDEXMANAGER_FACET_THREAD_COUNT = "blur.indexmanager.facet.thread.count"; public static final String BLUR_SHARD_DATA_FETCH_THREAD_COUNT = "blur.shard.data.fetch.thread.count"; - public static final String BLUR_SHARD_WARMUP_THREAD_COUNT = "blur.shard.warmup.thread.count"; - public static final String BLUR_SHARD_INDEX_WARMUP_THROTTLE = "blur.shard.index.warmup.throttle"; public static final String BLUR_MAX_CLAUSE_COUNT = "blur.max.clause.count"; public static final String BLUR_SHARD_CACHE_MAX_QUERYCACHE_ELEMENTS = "blur.shard.cache.max.querycache.elements"; public static final String BLUR_SHARD_OPENER_THREAD_COUNT = "blur.shard.opener.thread.count"; @@ -100,7 +98,6 @@ public class BlurConstants { public static final String BLUR_SHARD_THRIFT_MAX_READ_BUFFER_BYTES = "blur.shard.thrift.max.read.buffer.bytes"; public static final String BLUR_SHARD_THRIFT_ACCEPT_QUEUE_SIZE_PER_THREAD = "blur.shard.thrift.accept.queue.size.per.thread"; public static final String BLUR_SHARD_DISTRIBUTED_LAYOUT_FACTORY_CLASS = "blur.shard.distributed.layout.factory.class"; - public static final String BLUR_SHARD_WARMUP_DISABLED = "blur.shard.warmup.disabled"; public static final String BLUR_SHARD_DEEP_PAGING_CACHE_SIZE = "blur.shard.deep.paging.cache.size"; public static final String BLUR_SHARD_BLOCK_CACHE_V2_READ_CACHE_EXT = "blur.shard.block.cache.v2.read.cache.ext"; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e9dce540/blur-util/src/main/resources/blur-default.properties ---------------------------------------------------------------------- diff --git a/blur-util/src/main/resources/blur-default.properties b/blur-util/src/main/resources/blur-default.properties index 008649e..1fa82cf 100644 --- a/blur-util/src/main/resources/blur-default.properties +++ b/blur-util/src/main/resources/blur-default.properties @@ -57,6 +57,9 @@ blur.shard.command.driver.threads=16 # The number of command worker threads. blur.shard.command.worker.threads=16 +# The number of internal lucene worker threads used to make the search call more parallel. +blur.shard.internal.search.thread.count=16 + # The number of fetcher threads blur.shard.data.fetch.thread.count=8 @@ -84,15 +87,6 @@ blur.shard.cache.max.timetolive=60000 # Default implementation of the blur cache filter, which is a pass through filter that does nothing blur.shard.filter.cache.class=org.apache.blur.manager.DefaultBlurFilterCache -# Globally disable index warmup. -blur.shard.warmup.disabled=true - -# Default Blur index warmup class that warms the fields provided in the table descriptor -blur.shard.index.warmup.class=org.apache.blur.manager.indexserver.DefaultBlurIndexWarmup - -# Throttles the warmup to 30MB/s across all the warmup threads -blur.shard.index.warmup.throttle=30000000 - # By default the v2 version of the block cache is enabled blur.shard.block.cache.version=v2 @@ -175,9 +169,6 @@ blur.indexmanager.mutate.thread.count=8 # The number of thread used for parallel faceting in the index manager blur.indexmanager.facet.thread.count=8 -# Number of threads used for warming up the index -blur.shard.warmup.thread.count=8 - # The fetch count per Lucene search, this fetches pointers to hits blur.shard.fetchcount=110
