Updated Branches: refs/heads/apache-blur-0.2 01b4c7606 -> 7968b8f6b
More updates, making the quiet option during merges not effect the cache. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/7968b8f6 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/7968b8f6 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/7968b8f6 Branch: refs/heads/apache-blur-0.2 Commit: 7968b8f6b1bed16a24717e4f0cada96fc47fc572 Parents: 01b4c76 Author: Aaron McCurry <[email protected]> Authored: Tue Jan 7 10:41:12 2014 -0500 Committer: Aaron McCurry <[email protected]> Committed: Tue Jan 7 10:41:12 2014 -0500 ---------------------------------------------------------------------- .../store/blockcache_v2/CacheIndexInput.java | 46 +++++++++++++++++--- .../store/blockcache_v2/CacheIndexOutput.java | 22 ++++++---- .../cachevalue/UnsafeCacheValue.java | 5 ++- .../src/main/resources/blur-default.properties | 2 +- 4 files changed, 58 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7968b8f6/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexInput.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexInput.java b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexInput.java index df81832..35a03e6 100644 --- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexInput.java +++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexInput.java @@ -38,6 +38,7 @@ public class CacheIndexInput extends IndexInput { private IndexInput _indexInput; private CacheKey _key = new CacheKey(); private CacheValue _cacheValue; + private CacheValue _cacheValueQuietRef; private long _position; private int _blockPosition; @@ -210,6 +211,11 @@ public class CacheIndexInput extends IndexInput { _indexInput.close(); releaseCache(); } + if (_cacheValueQuietRef != null) { + CacheValue ref = _cacheValueQuietRef; + _cacheValueQuietRef = null; + ref.release(); + } } @Override @@ -260,6 +266,7 @@ public class CacheIndexInput extends IndexInput { if (clone._cacheValue != null) { clone._cacheValue.incRef(); } + clone._cacheValueQuietRef = null; return clone; } @@ -295,9 +302,37 @@ public class CacheIndexInput extends IndexInput { } } - private void fill() throws IOException { + private void fillQuietly() throws IOException { _key.setBlockId(getBlockId()); - _cacheValue = get(_key); + _cacheValue = _cache.getQuietly(_key); + if (_cacheValue == null) { + if (_cacheValueQuietRef == null) { + _cacheValueQuietRef = _cache.newInstance(_directory, _fileName); + } + _cacheValue = _cacheValueQuietRef; + _cacheValue.incRef(); + long filePosition = getFilePosition(); + _indexInput.seek(filePosition); + byte[] buffer = _store.takeBuffer(_bufferSize); + int len = (int) Math.min(_cacheBlockSize, _fileLength - filePosition); + int cachePosition = 0; + while (len > 0) { + int length = Math.min(_bufferSize, len); + _indexInput.readBytes(buffer, 0, length); + _cacheValue.write(cachePosition, buffer, 0, length); + len -= length; + cachePosition += length; + } + _store.putBuffer(buffer); + } else { + _cacheValue.incRef(); + } + _blockPosition = getBlockPosition(); + } + + private void fillNormally() throws IOException { + _key.setBlockId(getBlockId()); + _cacheValue = _cache.get(_key); if (_cacheValue == null) { _cacheValue = _cache.newInstance(_directory, _fileName); _cacheValue.incRef(); @@ -321,11 +356,12 @@ public class CacheIndexInput extends IndexInput { _blockPosition = getBlockPosition(); } - private CacheValue get(CacheKey key) { + private void fill() throws IOException { if (_quiet) { - return _cache.getQuietly(key); + fillQuietly(); + } else { + fillNormally(); } - return _cache.get(key); } private int getBlockPosition() { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7968b8f6/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java index 7c15cf2..dd0dc29 100644 --- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java +++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java @@ -33,6 +33,7 @@ public class CacheIndexOutput extends IndexOutput { private final int _fileBufferSize; private final int _cacheBlockSize; private final Store _store; + private final boolean _shouldBeQuiet; private long _position; private byte[] _buffer; @@ -49,6 +50,7 @@ public class CacheIndexOutput extends IndexOutput { _indexOutput = indexOutput; _store = BufferStore.instance(_cacheBlockSize); _buffer = _store.takeBuffer(_cacheBlockSize); + _shouldBeQuiet = _cache.shouldBeQuiet(directory, fileName); } @Override @@ -89,19 +91,21 @@ public class CacheIndexOutput extends IndexOutput { } private void flushInternal() throws IOException { - int length = _cacheBlockSize - remaining(); + final int length = _cacheBlockSize - remaining(); if (length == 0) { return; } - CacheValue cacheValue = _cache.newInstance(_directory, _fileName); - cacheValue.incRef(); - writeBufferToOutputStream(length); - cacheValue.write(0, _buffer, 0, length); - long blockId = (_position - length) / _cacheBlockSize; - cacheValue = cacheValue.trim(length); - _cache.put(new CacheKey(_fileId, blockId), cacheValue); - cacheValue.decRef(); + if (!_shouldBeQuiet) { + CacheValue cacheValue = _cache.newInstance(_directory, _fileName); + cacheValue.incRef(); + cacheValue.write(0, _buffer, 0, length); + long blockId = (_position - length) / _cacheBlockSize; + cacheValue = cacheValue.trim(length); + _cache.put(new CacheKey(_fileId, blockId), cacheValue); + cacheValue.decRef(); + } _bufferPosition = 0; + writeBufferToOutputStream(length); } private void writeBufferToOutputStream(int len) throws IOException { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7968b8f6/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeCacheValue.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeCacheValue.java b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeCacheValue.java index 5aa4436..6f230fd 100644 --- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeCacheValue.java +++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeCacheValue.java @@ -96,8 +96,9 @@ public class UnsafeCacheValue extends BaseCacheValue { _unsafe.freeMemory(_address); _released = true; _offHeapMemorySize.addAndGet(0 - _capacity); - } else { - new Throwable().printStackTrace(); +// } else { + // @TODO this is here to debug against double releases. +// new Throwable().printStackTrace(); } } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7968b8f6/blur-util/src/main/resources/blur-default.properties ---------------------------------------------------------------------- diff --git a/blur-util/src/main/resources/blur-default.properties b/blur-util/src/main/resources/blur-default.properties index e0d6180..c0f71c3 100644 --- a/blur-util/src/main/resources/blur-default.properties +++ b/blur-util/src/main/resources/blur-default.properties @@ -146,7 +146,7 @@ blur.shard.time.between.commits=30000 blur.shard.time.between.refreshs=3000 # The max number of threads used during index merges -blur.shard.merge.thread.count=3 +blur.shard.merge.thread.count=8 # The maximum number of clauses in a BooleanQuery blur.max.clause.count=1024
