Repository: incubator-blur Updated Branches: refs/heads/master a9fd70527 -> eec20f6ca
Working on BLUR-398. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/eec20f6c Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/eec20f6c Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/eec20f6c Branch: refs/heads/master Commit: eec20f6ca518697f66a743e42b8711f917fd310d Parents: a9fd705 Author: Aaron McCurry <[email protected]> Authored: Sat Dec 27 12:38:26 2014 -0500 Committer: Aaron McCurry <[email protected]> Committed: Sat Dec 27 12:38:26 2014 -0500 ---------------------------------------------------------------------- .../store/BlockCacheDirectoryFactoryV2.java | 7 +- .../blur/store/blockcache_v2/BaseCache.java | 10 +- .../apache/blur/store/blockcache_v2/Cache.java | 14 +-- .../store/blockcache_v2/CacheDirectory.java | 2 +- .../store/blockcache_v2/CacheIndexInput.java | 6 +- .../store/blockcache_v2/CacheIndexOutput.java | 2 +- .../store/blockcache_v2/CachePoolStrategy.java | 27 +++++ .../blur/store/blockcache_v2/PooledCache.java | 116 +++++++++++++++++++ .../blockcache_v2/SingleCachePoolStrategy.java | 45 +++++++ .../blockcache_v2/CacheIndexInputTest.java | 9 +- 10 files changed, 215 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/eec20f6c/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java b/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java index 82e0a93..c1b6b72 100644 --- a/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java +++ b/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java @@ -45,8 +45,11 @@ import org.apache.blur.store.blockcache_v2.BaseCache; import org.apache.blur.store.blockcache_v2.BaseCache.STORE; import org.apache.blur.store.blockcache_v2.Cache; import org.apache.blur.store.blockcache_v2.CacheDirectory; +import org.apache.blur.store.blockcache_v2.CachePoolStrategy; import org.apache.blur.store.blockcache_v2.FileNameFilter; +import org.apache.blur.store.blockcache_v2.PooledCache; import org.apache.blur.store.blockcache_v2.Quiet; +import org.apache.blur.store.blockcache_v2.SingleCachePoolStrategy; import org.apache.blur.store.blockcache_v2.Size; import org.apache.lucene.store.Directory; @@ -160,7 +163,9 @@ public class BlockCacheDirectoryFactoryV2 extends BlockCacheDirectoryFactory { } }; - _cache = new BaseCache(totalNumberOfBytes, fileBufferSize, cacheBlockSize, readFilter, writeFilter, quiet, store); + BaseCache baseCache = new BaseCache(totalNumberOfBytes, fileBufferSize, cacheBlockSize, readFilter, writeFilter, quiet, store); + CachePoolStrategy cachePoolStrategy = new SingleCachePoolStrategy(baseCache); + _cache = new PooledCache(cachePoolStrategy); } private Set<String> getSet(String value) { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/eec20f6c/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java index d98637b..3457bbb 100644 --- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java +++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java @@ -258,7 +258,7 @@ public class BaseCache extends Cache implements Closeable { } @Override - public CacheValue get(CacheKey key) { + public CacheValue get(CacheDirectory directory, String fileName, CacheKey key) { CacheValue cacheValue = _cacheMap.get(key); if (cacheValue == null) { _misses.mark(); @@ -269,7 +269,7 @@ public class BaseCache extends Cache implements Closeable { } @Override - public CacheValue getQuietly(CacheKey key) { + public CacheValue getQuietly(CacheDirectory directory, String fileName, CacheKey key) { CacheValue cacheValue = _cacheMap.getQuietly(key); if (cacheValue != null) { _hits.mark(); @@ -278,7 +278,7 @@ public class BaseCache extends Cache implements Closeable { } @Override - public void put(CacheKey key, CacheValue value) { + public void put(CacheDirectory directory, String fileName, CacheKey key, CacheValue value) { CacheValue cacheValue = _cacheMap.put(key, value); if (cacheValue != null) { _evictions.mark(); @@ -286,13 +286,13 @@ public class BaseCache extends Cache implements Closeable { } @Override - public void releaseDirectory(String directoryName) { + public void releaseDirectory(CacheDirectory directory) { Set<Entry<FileIdKey, Long>> entrySet = _fileNameToId.entrySet(); Iterator<Entry<FileIdKey, Long>> iterator = entrySet.iterator(); while (iterator.hasNext()) { Entry<FileIdKey, Long> entry = iterator.next(); FileIdKey fileIdKey = entry.getKey(); - if (fileIdKey._directoryName.equals(directoryName)) { + if (fileIdKey._directoryName.equals(directory.getDirectoryName())) { iterator.remove(); } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/eec20f6c/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/Cache.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/Cache.java b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/Cache.java index 402c55c..8a59141 100644 --- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/Cache.java +++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/Cache.java @@ -121,7 +121,7 @@ public abstract class Cache implements Closeable { * the key. * @return the cache value or null. */ - public abstract CacheValue get(CacheKey key); + public abstract CacheValue get(CacheDirectory directory, String fileName, CacheKey key); /** * Gets the cache value for the given key. Null if missing. NOTE: This method @@ -131,7 +131,7 @@ public abstract class Cache implements Closeable { * the key. * @return the cache value or null. */ - public abstract CacheValue getQuietly(CacheKey key); + public abstract CacheValue getQuietly(CacheDirectory directory, String fileName, CacheKey key); /** * Puts the cache entry into the cache. @@ -141,7 +141,7 @@ public abstract class Cache implements Closeable { * @param value * the value. */ - public abstract void put(CacheKey key, CacheValue value); + public abstract void put(CacheDirectory directory, String fileName, CacheKey key, CacheValue value); /** * Removes the file from the cache. @@ -157,10 +157,10 @@ public abstract class Cache implements Closeable { /** * This is called when the CacheDirectory is finalized. * - * @param directoryName - * the directory name. + * @param directory + * the directory. */ - public abstract void releaseDirectory(String directoryName); + public abstract void releaseDirectory(CacheDirectory directory); /** * Determines if the reader should be quiet or not. @@ -182,7 +182,7 @@ public abstract class Cache implements Closeable { * * @param fileId * the file id. - * @throws IOException + * @throws IOException */ public abstract void fileClosedForWriting(CacheDirectory directory, String fileName, long fileId) throws IOException; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/eec20f6c/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheDirectory.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheDirectory.java b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheDirectory.java index 04f68d2..24b31aa 100644 --- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheDirectory.java +++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheDirectory.java @@ -121,7 +121,7 @@ public class CacheDirectory extends Directory implements DirectoryDecorator, Las } public void close() throws IOException { - _cache.releaseDirectory(getDirectoryName()); + _cache.releaseDirectory(this); _internal.close(); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/eec20f6c/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexInput.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexInput.java b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexInput.java index dfdbccc..7ea9537 100644 --- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexInput.java +++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexInput.java @@ -389,7 +389,7 @@ public class CacheIndexInput extends IndexInput { private void fillQuietly() throws IOException { _key.setBlockId(getBlockId()); - _cacheValue = _cache.getQuietly(_key); + _cacheValue = _cache.getQuietly(_directory, _fileName, _key); if (_cacheValue == null) { if (_cacheValueQuietRefCannotBeReleased == null) { // @TODO this could be improved. @@ -416,7 +416,7 @@ public class CacheIndexInput extends IndexInput { private void fillNormally() throws IOException { _key.setBlockId(getBlockId()); - _cacheValue = _cache.get(_key); + _cacheValue = _cache.get(_directory, _fileName, _key); if (_cacheValue == null) { _cacheValue = _cache.newInstance(_directory, _fileName); long filePosition = getFilePosition(); @@ -432,7 +432,7 @@ public class CacheIndexInput extends IndexInput { cachePosition += length; } _store.putBuffer(buffer); - _cache.put(_key.clone(), _cacheValue); + _cache.put(_directory, _fileName, _key.clone(), _cacheValue); } _blockPosition = getBlockPosition(); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/eec20f6c/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java index 120dc81..13d3aac 100644 --- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java +++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java @@ -100,7 +100,7 @@ public class CacheIndexOutput extends IndexOutput { cacheValue.write(0, _buffer, 0, length); long blockId = (_position - length) / _cacheBlockSize; cacheValue = cacheValue.trim(length); - _cache.put(new CacheKey(_fileId, blockId), cacheValue); + _cache.put(_directory, _fileName, new CacheKey(_fileId, blockId), cacheValue); } _bufferPosition = 0; writeBufferToOutputStream(length); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/eec20f6c/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CachePoolStrategy.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CachePoolStrategy.java b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CachePoolStrategy.java new file mode 100644 index 0000000..dcfc62b --- /dev/null +++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CachePoolStrategy.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.store.blockcache_v2; + +import java.io.Closeable; + +public abstract class CachePoolStrategy implements Closeable { + + public abstract Cache getCache(CacheDirectory directory, String fileName); + + public abstract Iterable<Cache> getCaches(CacheDirectory directory); + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/eec20f6c/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/PooledCache.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/PooledCache.java b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/PooledCache.java new file mode 100644 index 0000000..2527332 --- /dev/null +++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/PooledCache.java @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.store.blockcache_v2; + +import java.io.IOException; + +import org.apache.lucene.store.IOContext; + +public class PooledCache extends Cache { + + private final CachePoolStrategy _cachePoolStrategy; + + public PooledCache(CachePoolStrategy cachePoolStrategy) { + _cachePoolStrategy = cachePoolStrategy; + } + + @Override + public void close() throws IOException { + _cachePoolStrategy.close(); + } + + @Override + public CacheValue newInstance(CacheDirectory directory, String fileName, int cacheBlockSize) { + Cache cache = _cachePoolStrategy.getCache(directory, fileName); + return cache.newInstance(directory, fileName, cacheBlockSize); + } + + @Override + public long getFileId(CacheDirectory directory, String fileName) throws IOException { + Cache cache = _cachePoolStrategy.getCache(directory, fileName); + return cache.getFileId(directory, fileName); + } + + @Override + public int getCacheBlockSize(CacheDirectory directory, String fileName) { + Cache cache = _cachePoolStrategy.getCache(directory, fileName); + return cache.getCacheBlockSize(directory, fileName); + } + + @Override + public int getFileBufferSize(CacheDirectory directory, String fileName) { + Cache cache = _cachePoolStrategy.getCache(directory, fileName); + return cache.getFileBufferSize(directory, fileName); + } + + @Override + public boolean cacheFileForReading(CacheDirectory directory, String fileName, IOContext context) { + Cache cache = _cachePoolStrategy.getCache(directory, fileName); + return cache.cacheFileForReading(directory, fileName, context); + } + + @Override + public boolean cacheFileForWriting(CacheDirectory directory, String fileName, IOContext context) { + Cache cache = _cachePoolStrategy.getCache(directory, fileName); + return cache.cacheFileForWriting(directory, fileName, context); + } + + @Override + public CacheValue get(CacheDirectory directory, String fileName, CacheKey key) { + Cache cache = _cachePoolStrategy.getCache(directory, fileName); + return cache.get(directory, fileName, key); + } + + @Override + public CacheValue getQuietly(CacheDirectory directory, String fileName, CacheKey key) { + Cache cache = _cachePoolStrategy.getCache(directory, fileName); + return cache.getQuietly(directory, fileName, key); + } + + @Override + public void put(CacheDirectory directory, String fileName, CacheKey key, CacheValue value) { + Cache cache = _cachePoolStrategy.getCache(directory, fileName); + cache.put(directory, fileName, key, value); + } + + @Override + public void removeFile(CacheDirectory directory, String fileName) throws IOException { + Cache cache = _cachePoolStrategy.getCache(directory, fileName); + cache.removeFile(directory, fileName); + } + + @Override + public void releaseDirectory(CacheDirectory directory) { + Iterable<Cache> caches = _cachePoolStrategy.getCaches(directory); + for (Cache cache : caches) { + cache.releaseDirectory(directory); + } + } + + @Override + public boolean shouldBeQuiet(CacheDirectory directory, String fileName) { + Cache cache = _cachePoolStrategy.getCache(directory, fileName); + return cache.shouldBeQuiet(directory, fileName); + } + + @Override + public void fileClosedForWriting(CacheDirectory directory, String fileName, long fileId) throws IOException { + Cache cache = _cachePoolStrategy.getCache(directory, fileName); + cache.fileClosedForWriting(directory, fileName, fileId); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/eec20f6c/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/SingleCachePoolStrategy.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/SingleCachePoolStrategy.java b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/SingleCachePoolStrategy.java new file mode 100644 index 0000000..7420f63 --- /dev/null +++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/SingleCachePoolStrategy.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.store.blockcache_v2; + +import java.io.IOException; +import java.util.Arrays; + +public class SingleCachePoolStrategy extends CachePoolStrategy { + + private final Cache _cache; + + public SingleCachePoolStrategy(Cache cache) { + _cache = cache; + } + + @Override + public void close() throws IOException { + _cache.close(); + } + + @Override + public Cache getCache(CacheDirectory directory, String fileName) { + return _cache; + } + + @Override + public Iterable<Cache> getCaches(CacheDirectory directory) { + return Arrays.asList(_cache); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/eec20f6c/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheIndexInputTest.java ---------------------------------------------------------------------- diff --git a/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheIndexInputTest.java b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheIndexInputTest.java index a87f28f..ed50696 100644 --- a/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheIndexInputTest.java +++ b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheIndexInputTest.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.util.Random; import org.apache.blur.store.blockcache_v2.cachevalue.ByteArrayCacheValue; -import org.apache.blur.store.blockcache_v2.cachevalue.UnsafeCacheValue; import org.apache.blur.store.buffer.BufferStore; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; @@ -227,12 +226,12 @@ public class CacheIndexInputTest { } @Override - public CacheValue get(CacheKey key) { + public CacheValue get(CacheDirectory directory, String fileName, CacheKey key) { return cache.get(key); } @Override - public void put(CacheKey key, CacheValue value) { + public void put(CacheDirectory directory, String fileName, CacheKey key, CacheValue value) { cache.put(key, value); } @@ -242,12 +241,12 @@ public class CacheIndexInputTest { } @Override - public void releaseDirectory(String directoryName) { + public void releaseDirectory(CacheDirectory directory) { } @Override - public CacheValue getQuietly(CacheKey key) { + public CacheValue getQuietly(CacheDirectory directory, String fileName, CacheKey key) { return cache.getQuietly(key); }
