ACCUMULO-4644 Removed AccumuloConfiguration from BlockCacheManager API. Also moved everything not related to the BlockCacheManager API out of the o.a.a.core.file.blockfile.cache package.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d877a2df Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d877a2df Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d877a2df Branch: refs/heads/master Commit: d877a2df2943e48d70d99b96616844d0dff9a501 Parents: 628fbaf Author: Keith Turner <[email protected]> Authored: Fri May 26 16:29:17 2017 -0400 Committer: Keith Turner <[email protected]> Committed: Fri May 26 16:29:17 2017 -0400 ---------------------------------------------------------------------- .../core/client/rfile/RFileScanner.java | 6 +- .../cache/BlockCacheConfiguration.java | 91 ---------- .../file/blockfile/cache/BlockCacheManager.java | 95 +++++++--- .../core/file/blockfile/cache/CachedBlock.java | 122 ------------- .../file/blockfile/cache/CachedBlockQueue.java | 120 ------------- .../core/file/blockfile/cache/ClassSize.java | 172 ------------------- .../core/file/blockfile/cache/HeapSize.java | 42 ----- .../file/blockfile/cache/SizeConstants.java | 58 ------- .../cache/impl/BlockCacheConfiguration.java | 97 +++++++++++ .../cache/impl/BlockCacheManagerFactory.java | 59 +++++++ .../file/blockfile/cache/impl/ClassSize.java | 172 +++++++++++++++++++ .../blockfile/cache/impl/SizeConstants.java | 58 +++++++ .../file/blockfile/cache/lru/CachedBlock.java | 126 ++++++++++++++ .../blockfile/cache/lru/CachedBlockQueue.java | 120 +++++++++++++ .../core/file/blockfile/cache/lru/HeapSize.java | 42 +++++ .../file/blockfile/cache/lru/LruBlockCache.java | 7 +- .../cache/lru/LruBlockCacheConfiguration.java | 34 +++- .../cache/lru/LruBlockCacheManager.java | 3 +- .../cache/tinylfu/TinyLfuBlockCache.java | 17 +- .../tinylfu/TinyLfuBlockCacheConfiguration.java | 32 ---- .../cache/tinylfu/TinyLfuBlockCacheManager.java | 8 +- .../blockfile/cache/BlockCacheFactoryTest.java | 10 +- .../blockfile/cache/BlockCacheManagerTest.java | 33 ++++ .../cache/BlockConfigurationHelperTest.java | 32 ---- .../blockfile/cache/TestCachedBlockQueue.java | 8 +- .../file/blockfile/cache/TestLruBlockCache.java | 40 +++-- .../accumulo/core/file/rfile/RFileTest.java | 6 +- .../tserver/TabletServerResourceManager.java | 6 +- 28 files changed, 866 insertions(+), 750 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/d877a2df/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java index 889493a..e10c073 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java +++ b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java @@ -46,6 +46,8 @@ import org.apache.accumulo.core.file.blockfile.cache.BlockCache; import org.apache.accumulo.core.file.blockfile.cache.BlockCacheManager; import org.apache.accumulo.core.file.blockfile.cache.CacheEntry; import org.apache.accumulo.core.file.blockfile.cache.CacheType; +import org.apache.accumulo.core.file.blockfile.cache.impl.BlockCacheConfiguration; +import org.apache.accumulo.core.file.blockfile.cache.impl.BlockCacheManagerFactory; import org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile; import org.apache.accumulo.core.file.rfile.RFile; import org.apache.accumulo.core.file.rfile.RFile.Reader; @@ -148,14 +150,14 @@ class RFileScanner extends ScannerOptions implements Scanner { } try { - blockCacheManager = BlockCacheManager.getClientInstance(cc); + blockCacheManager = BlockCacheManagerFactory.getClientInstance(cc); if (opts.indexCacheSize > 0) { cc.set(Property.TSERV_INDEXCACHE_SIZE, Long.toString(opts.indexCacheSize)); } if (opts.dataCacheSize > 0) { cc.set(Property.TSERV_DATACACHE_SIZE, Long.toString(opts.dataCacheSize)); } - blockCacheManager.start(cc); + blockCacheManager.start(new BlockCacheConfiguration(cc)); this.indexCache = blockCacheManager.getBlockCache(CacheType.INDEX); this.dataCache = blockCacheManager.getBlockCache(CacheType.DATA); } catch (RuntimeException e) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/d877a2df/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/BlockCacheConfiguration.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/BlockCacheConfiguration.java b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/BlockCacheConfiguration.java deleted file mode 100644 index efab628..0000000 --- a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/BlockCacheConfiguration.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.core.file.blockfile.cache; - -import java.util.Map; -import java.util.Optional; - -import org.apache.accumulo.core.conf.AccumuloConfiguration; -import org.apache.accumulo.core.conf.Property; - -public class BlockCacheConfiguration { - - public static final String CACHE_PROPERTY_BASE = Property.GENERAL_ARBITRARY_PROP_PREFIX + "cache.block."; - - /** Maximum allowable size of cache (block put if size > max, evict) */ - private final long maxSize; - - /** Approximate block size */ - private final long blockSize; - - private final Map<String,String> genProps; - - private final String prefix; - - private final String defaultPrefix; - - public BlockCacheConfiguration(AccumuloConfiguration conf, CacheType type, String implName) { - defaultPrefix = getDefaultPrefix(implName); - prefix = getPrefix(type, implName); - genProps = conf.getAllPropertiesWithPrefix(Property.GENERAL_ARBITRARY_PROP_PREFIX); - - switch (type) { - case INDEX: - this.maxSize = conf.getAsBytes(Property.TSERV_INDEXCACHE_SIZE); - break; - case DATA: - this.maxSize = conf.getAsBytes(Property.TSERV_DATACACHE_SIZE); - break; - case SUMMARY: - this.maxSize = conf.getAsBytes(Property.TSERV_SUMMARYCACHE_SIZE); - break; - default: - throw new IllegalArgumentException("Unknown block cache type"); - } - this.blockSize = conf.getAsBytes(Property.TSERV_DEFAULT_BLOCKSIZE); - } - - public long getMaxSize() { - return this.maxSize; - } - - public long getBlockSize() { - return this.blockSize; - } - - protected Optional<String> get(String suffix) { - String val = genProps.get(prefix + suffix); - if (val == null) { - val = genProps.get(defaultPrefix + suffix); - } - return Optional.ofNullable(val); - } - - public static String getDefaultPrefix(String implName) { - return CACHE_PROPERTY_BASE + implName + ".default."; - } - - public static String getPrefix(CacheType type, String implName) { - return CACHE_PROPERTY_BASE + implName + "." + type.name().toLowerCase() + "."; - } - - @Override - public String toString() { - return "maxSize: " + getMaxSize() + ", blockSize: " + getBlockSize(); - } -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/d877a2df/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/BlockCacheManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/BlockCacheManager.java b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/BlockCacheManager.java index be77ee2..f19167c 100644 --- a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/BlockCacheManager.java +++ b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/BlockCacheManager.java @@ -20,25 +20,75 @@ package org.apache.accumulo.core.file.blockfile.cache; import java.util.HashMap; import java.util.Map; -import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; -import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public abstract class BlockCacheManager { - private static final Logger LOG = LoggerFactory.getLogger(BlockCacheManager.class); + public static final Logger LOG = LoggerFactory.getLogger(BlockCacheManager.class); private final Map<CacheType,BlockCache> caches = new HashMap<>(); + public static final String CACHE_PROPERTY_BASE = Property.GENERAL_ARBITRARY_PROP_PREFIX + "cache."; + + public static interface Configuration { + + /** + * Before Accumulo's cache implementation was configurable, its built in caches had a configurable size. These sizes were specified by the system properties + * {@code tserver.cache.data.size}, {@code tserver.cache.index.size}, and {code tserver.cache.summary.size}. This method returns the values of those + * settings. The settings are made available, but cache implementations are under no obligation to use them. + * + */ + long getMaxSize(CacheType type); + + /** + * Before Accumulo's cache implementation was configurable, its built in cache had a configurable block size. This block size was specified by the system + * property {@code tserver.default.blocksize}. This method returns the value of that setting. The setting is made available, but cache implementations are + * under no obligation to use it. + * + */ + long getBlockSize(); + + /** + * This method provides a way for a cache implementation to access arbitrary configuration set by a user. + * + * <p> + * Returns all Accumulo properties that have a prefix of {@code general.custom.cache.<prefix>.<type>.} or {@code general.custom.cache.<prefix>.default.} + * with values for specific cache types overriding defaults. + * + * <p> + * For example assume the following data is in Accumulo's system config. + * + * <pre> + * general.custom.cache.lru.default.evictAfter=3600 + * general.custom.cache.lru.default.loadFactor=.75 + * general.custom.cache.lru.index.loadFactor=.55 + * general.custom.cache.lru.data.loadFactor=.65 + * </pre> + * + * <p> + * If this method is called with {@code prefix=lru} and {@code type=INDEX} then it would return a map with the following key values. The load factor setting + * for index overrides the default value. + * + * <pre> + * evictAfter=3600 + * loadFactor=.55 + * </pre> + * + * @param prefix + * A unique identifier that corresponds to a particular BlockCacheManager implementation. + */ + Map<String,String> getProperties(String prefix, CacheType type); + } + /** * Initialize the caches for each CacheType based on the configuration * * @param conf * accumulo configuration */ - public void start(AccumuloConfiguration conf) { + public void start(Configuration conf) { for (CacheType type : CacheType.values()) { BlockCache cache = this.createCache(conf, type); this.caches.put(type, cache); @@ -70,38 +120,29 @@ public abstract class BlockCacheManager { * cache configuration * @return configured block cache */ - protected abstract BlockCache createCache(AccumuloConfiguration conf, CacheType type); + protected abstract BlockCache createCache(Configuration conf, CacheType type); /** - * Get the BlockCacheFactory specified by the property 'tserver.cache.factory.class' using the AccumuloVFSClassLoader + * A convenience method that returns a string of the from {@code general.custom.cache.<prefix>.default.} this method is useful for configuring a cache + * manager. * - * @param conf - * accumulo configuration - * @return block cache manager instance - * @throws Exception - * error loading block cache manager implementation class + * @param prefix + * A unique identifier that corresponds to a particular BlockCacheManager implementation. + * @see Configuration#getProperties(String, CacheType) */ - public static synchronized BlockCacheManager getInstance(AccumuloConfiguration conf) throws Exception { - String impl = conf.get(Property.TSERV_CACHE_MANAGER_IMPL); - Class<? extends BlockCacheManager> clazz = AccumuloVFSClassLoader.loadClass(impl, BlockCacheManager.class); - LOG.info("Created new block cache manager of type: {}", clazz.getSimpleName()); - return (BlockCacheManager) clazz.newInstance(); + public static String getFullyQualifiedPropertyPrefix(String prefix) { + return CACHE_PROPERTY_BASE + prefix + ".default."; } /** - * Get the BlockCacheFactory specified by the property 'tserver.cache.factory.class' + * A convenience method that returns a string of the from {@code general.custom.cache.<prefix>.<type>.} this method is useful for configuring a cache manager. * - * @param conf - * accumulo configuration - * @return block cache manager instance - * @throws Exception - * error loading block cache manager implementation class + * @param prefix + * A unique identifier that corresponds to a particular BlockCacheManager implementation. + * @see Configuration#getProperties(String, CacheType) */ - public static synchronized BlockCacheManager getClientInstance(AccumuloConfiguration conf) throws Exception { - String impl = conf.get(Property.TSERV_CACHE_MANAGER_IMPL); - Class<? extends BlockCacheManager> clazz = Class.forName(impl).asSubclass(BlockCacheManager.class); - LOG.info("Created new block cache factory of type: {}", clazz.getSimpleName()); - return (BlockCacheManager) clazz.newInstance(); + public static String getFullyQualifiedPropertyPrefix(String prefix, CacheType type) { + return CACHE_PROPERTY_BASE + prefix + "." + type.name().toLowerCase() + "."; } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/d877a2df/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/CachedBlock.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/CachedBlock.java b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/CachedBlock.java deleted file mode 100644 index 25c5c95..0000000 --- a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/CachedBlock.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.core.file.blockfile.cache; - -import java.util.Objects; - -/** - * Represents an entry in the configurable block cache. - * - * <p> - * Makes the block memory-aware with {@link HeapSize} and Comparable to sort by access time for the LRU. It also takes care of priority by either instantiating - * as in-memory or handling the transition from single to multiple access. - */ -public class CachedBlock implements HeapSize, Comparable<CachedBlock>, CacheEntry { - - public final static long PER_BLOCK_OVERHEAD = ClassSize.align(ClassSize.OBJECT + (3 * ClassSize.REFERENCE) + (2 * SizeConstants.SIZEOF_LONG) - + ClassSize.STRING + ClassSize.BYTE_BUFFER); - - public static enum BlockPriority { - /** - * Accessed a single time (used for scan-resistance) - */ - SINGLE, - /** - * Accessed multiple times - */ - MULTI, - /** - * Block from in-memory store - */ - MEMORY - } - - private final String blockName; - private final byte buf[]; - private volatile long accessTime; - private long size; - private BlockPriority priority; - private Object index; - - public CachedBlock(String blockName, byte buf[], long accessTime, boolean inMemory) { - this.blockName = blockName; - this.buf = buf; - this.accessTime = accessTime; - this.size = ClassSize.align(blockName.length()) + ClassSize.align(buf.length) + PER_BLOCK_OVERHEAD; - if (inMemory) { - this.priority = BlockPriority.MEMORY; - } else { - this.priority = BlockPriority.SINGLE; - } - } - - /** - * Block has been accessed. Update its local access time. - */ - public void access(long accessTime) { - this.accessTime = accessTime; - if (this.priority == BlockPriority.SINGLE) { - this.priority = BlockPriority.MULTI; - } - } - - @Override - public long heapSize() { - return size; - } - - @Override - public int hashCode() { - return Objects.hashCode(accessTime); - } - - @Override - public boolean equals(Object obj) { - return this == obj || (obj != null && obj instanceof CachedBlock && 0 == compareTo((CachedBlock) obj)); - } - - @Override - public int compareTo(CachedBlock that) { - if (this.accessTime == that.accessTime) - return 0; - return this.accessTime < that.accessTime ? 1 : -1; - } - - @Override - public byte[] getBuffer() { - return this.buf; - } - - public String getName() { - return this.blockName; - } - - public BlockPriority getPriority() { - return this.priority; - } - - @Override - public Object getIndex() { - return index; - } - - @Override - public void setIndex(Object idx) { - this.index = idx; - } -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/d877a2df/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/CachedBlockQueue.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/CachedBlockQueue.java b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/CachedBlockQueue.java deleted file mode 100644 index 248634d..0000000 --- a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/CachedBlockQueue.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.core.file.blockfile.cache; - -import java.util.LinkedList; -import java.util.PriorityQueue; - -/** - * A memory-bound queue that will grow until an element brings total size >= maxSize. From then on, only entries that are sorted larger than the smallest - * current entry will be inserted/replaced. - * - * <p> - * Use this when you want to find the largest elements (according to their ordering, not their heap size) that consume as close to the specified maxSize as - * possible. Default behavior is to grow just above rather than just below specified max. - * - * <p> - * Object used in this queue must implement {@link HeapSize} as well as {@link Comparable}. - */ -public class CachedBlockQueue implements HeapSize { - - private PriorityQueue<CachedBlock> queue; - - private long heapSize; - private long maxSize; - - /** - * @param maxSize - * the target size of elements in the queue - * @param blockSize - * expected average size of blocks - */ - public CachedBlockQueue(long maxSize, long blockSize) { - int initialSize = (int) Math.ceil(maxSize / (double) blockSize); - if (initialSize == 0) - initialSize++; - queue = new PriorityQueue<>(initialSize); - heapSize = 0; - this.maxSize = maxSize; - } - - /** - * Attempt to add the specified cached block to this queue. - * - * <p> - * If the queue is smaller than the max size, or if the specified element is ordered before the smallest element in the queue, the element will be added to - * the queue. Otherwise, there is no side effect of this call. - * - * @param cb - * block to try to add to the queue - */ - public void add(CachedBlock cb) { - if (heapSize < maxSize) { - queue.add(cb); - heapSize += cb.heapSize(); - } else { - CachedBlock head = queue.peek(); - if (cb.compareTo(head) > 0) { - heapSize += cb.heapSize(); - heapSize -= head.heapSize(); - if (heapSize > maxSize) { - queue.poll(); - } else { - heapSize += head.heapSize(); - } - queue.add(cb); - } - } - } - - /** - * Get a sorted List of all elements in this queue, in descending order. - * - * @return list of cached elements in descending order - */ - public CachedBlock[] get() { - LinkedList<CachedBlock> blocks = new LinkedList<>(); - while (!queue.isEmpty()) { - blocks.addFirst(queue.poll()); - } - return blocks.toArray(new CachedBlock[blocks.size()]); - } - - /** - * Get a sorted List of all elements in this queue, in descending order. - * - * @return list of cached elements in descending order - */ - public LinkedList<CachedBlock> getList() { - LinkedList<CachedBlock> blocks = new LinkedList<>(); - while (!queue.isEmpty()) { - blocks.addFirst(queue.poll()); - } - return blocks; - } - - /** - * Total size of all elements in this queue. - * - * @return size of all elements currently in queue, in bytes - */ - @Override - public long heapSize() { - return heapSize; - } -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/d877a2df/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/ClassSize.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/ClassSize.java b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/ClassSize.java deleted file mode 100644 index 5394a77..0000000 --- a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/ClassSize.java +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.accumulo.core.file.blockfile.cache; - -import java.util.Properties; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -/** - * Class for determining the "size" of a class, an attempt to calculate the actual bytes that an object of this class will occupy in memory - * - * The core of this class is taken from the Derby project - */ -public class ClassSize { - static final Log LOG = LogFactory.getLog(ClassSize.class); - - /** Array overhead */ - public static final int ARRAY; - - /** Overhead for ArrayList(0) */ - public static final int ARRAYLIST; - - /** Overhead for ByteBuffer */ - public static final int BYTE_BUFFER; - - /** Overhead for an Integer */ - public static final int INTEGER; - - /** Overhead for entry in map */ - public static final int MAP_ENTRY; - - /** Object overhead is minimum 2 * reference size (8 bytes on 64-bit) */ - public static final int OBJECT; - - /** Reference size is 8 bytes on 64-bit, 4 bytes on 32-bit */ - public static final int REFERENCE; - - /** String overhead */ - public static final int STRING; - - /** Overhead for TreeMap */ - public static final int TREEMAP; - - /** Overhead for ConcurrentHashMap */ - public static final int CONCURRENT_HASHMAP; - - /** Overhead for ConcurrentHashMap.Entry */ - public static final int CONCURRENT_HASHMAP_ENTRY; - - /** Overhead for ConcurrentHashMap.Segment */ - public static final int CONCURRENT_HASHMAP_SEGMENT; - - /** Overhead for ConcurrentSkipListMap */ - public static final int CONCURRENT_SKIPLISTMAP; - - /** Overhead for ConcurrentSkipListMap Entry */ - public static final int CONCURRENT_SKIPLISTMAP_ENTRY; - - /** Overhead for ReentrantReadWriteLock */ - public static final int REENTRANT_LOCK; - - /** Overhead for AtomicLong */ - public static final int ATOMIC_LONG; - - /** Overhead for AtomicInteger */ - public static final int ATOMIC_INTEGER; - - /** Overhead for AtomicBoolean */ - public static final int ATOMIC_BOOLEAN; - - /** Overhead for CopyOnWriteArraySet */ - public static final int COPYONWRITE_ARRAYSET; - - /** Overhead for CopyOnWriteArrayList */ - public static final int COPYONWRITE_ARRAYLIST; - - private static final String THIRTY_TWO = "32"; - - /** - * Method for reading the arc settings and setting overheads according to 32-bit or 64-bit architecture. - */ - static { - // Figure out whether this is a 32 or 64 bit machine. - Properties sysProps = System.getProperties(); - String arcModel = sysProps.getProperty("sun.arch.data.model"); - - // Default value is set to 8, covering the case when arcModel is unknown - REFERENCE = arcModel.equals(THIRTY_TWO) ? 4 : 8; - - OBJECT = 2 * REFERENCE; - - ARRAY = 3 * REFERENCE; - - ARRAYLIST = align(OBJECT + align(REFERENCE) + align(ARRAY) + (2 * SizeConstants.SIZEOF_INT)); - - BYTE_BUFFER = align(OBJECT + align(REFERENCE) + align(ARRAY) + (5 * SizeConstants.SIZEOF_INT) + (3 * SizeConstants.SIZEOF_BOOLEAN) - + SizeConstants.SIZEOF_LONG); - - INTEGER = align(OBJECT + SizeConstants.SIZEOF_INT); - - MAP_ENTRY = align(OBJECT + 5 * REFERENCE + SizeConstants.SIZEOF_BOOLEAN); - - TREEMAP = align(OBJECT + (2 * SizeConstants.SIZEOF_INT) + align(7 * REFERENCE)); - - STRING = align(OBJECT + ARRAY + REFERENCE + 3 * SizeConstants.SIZEOF_INT); - - CONCURRENT_HASHMAP = align((2 * SizeConstants.SIZEOF_INT) + ARRAY + (6 * REFERENCE) + OBJECT); - - CONCURRENT_HASHMAP_ENTRY = align(REFERENCE + OBJECT + (3 * REFERENCE) + (2 * SizeConstants.SIZEOF_INT)); - - CONCURRENT_HASHMAP_SEGMENT = align(REFERENCE + OBJECT + (3 * SizeConstants.SIZEOF_INT) + SizeConstants.SIZEOF_FLOAT + ARRAY); - - CONCURRENT_SKIPLISTMAP = align(SizeConstants.SIZEOF_INT + OBJECT + (8 * REFERENCE)); - - CONCURRENT_SKIPLISTMAP_ENTRY = align(align(OBJECT + (3 * REFERENCE)) + /* one node per entry */ - align((OBJECT + (3 * REFERENCE)) / 2)); /* one index per two entries */ - - REENTRANT_LOCK = align(OBJECT + (3 * REFERENCE)); - - ATOMIC_LONG = align(OBJECT + SizeConstants.SIZEOF_LONG); - - ATOMIC_INTEGER = align(OBJECT + SizeConstants.SIZEOF_INT); - - ATOMIC_BOOLEAN = align(OBJECT + SizeConstants.SIZEOF_BOOLEAN); - - COPYONWRITE_ARRAYSET = align(OBJECT + REFERENCE); - - COPYONWRITE_ARRAYLIST = align(OBJECT + (2 * REFERENCE) + ARRAY); - } - - /** - * Aligns a number to 8. - * - * @param num - * number to align to 8 - * @return smallest number >= input that is a multiple of 8 - */ - public static int align(int num) { - return (int) (align((long) num)); - } - - /** - * Aligns a number to 8. - * - * @param num - * number to align to 8 - * @return smallest number >= input that is a multiple of 8 - */ - public static long align(long num) { - // The 7 comes from that the alignSize is 8 which is the number of bytes - // stored and sent together - return ((num + 7) >> 3) << 3; - } - -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/d877a2df/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/HeapSize.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/HeapSize.java b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/HeapSize.java deleted file mode 100644 index e6aef42..0000000 --- a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/HeapSize.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.core.file.blockfile.cache; - -/** - * Implementations can be asked for an estimate of their size in bytes. - * <p> - * Useful for sizing caches. Its a given that implementation approximations do not account for 32 vs 64 bit nor for different VM implementations. - * <p> - * An Object's size is determined by the non-static data members in it, as well as the fixed {@link Object} overhead. - * <p> - * For example: - * - * <pre> - * public class SampleObject implements HeapSize { - * int[] numbers; - * int x; - * } - * </pre> - */ -public interface HeapSize { - /** - * @return Approximate 'exclusive deep size' of implementing object. Includes count of payload and hosting object sizings. - */ - long heapSize(); - -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/d877a2df/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/SizeConstants.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/SizeConstants.java b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/SizeConstants.java deleted file mode 100644 index e5004cd..0000000 --- a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/SizeConstants.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.core.file.blockfile.cache; - -public class SizeConstants { - - public static final int SIZEOF_BOOLEAN = Byte.SIZE / Byte.SIZE; - - /** - * Size of byte in bytes - */ - public static final int SIZEOF_BYTE = SIZEOF_BOOLEAN; - - /** - * Size of char in bytes - */ - public static final int SIZEOF_CHAR = Character.SIZE / Byte.SIZE; - - /** - * Size of double in bytes - */ - public static final int SIZEOF_DOUBLE = Double.SIZE / Byte.SIZE; - - /** - * Size of float in bytes - */ - public static final int SIZEOF_FLOAT = Float.SIZE / Byte.SIZE; - - /** - * Size of int in bytes - */ - public static final int SIZEOF_INT = Integer.SIZE / Byte.SIZE; - - /** - * Size of long in bytes - */ - public static final int SIZEOF_LONG = Long.SIZE / Byte.SIZE; - - /** - * Size of short in bytes - */ - public static final int SIZEOF_SHORT = Short.SIZE / Byte.SIZE; - -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/d877a2df/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/impl/BlockCacheConfiguration.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/impl/BlockCacheConfiguration.java b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/impl/BlockCacheConfiguration.java new file mode 100644 index 0000000..7bf7503 --- /dev/null +++ b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/impl/BlockCacheConfiguration.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.file.blockfile.cache.impl; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.file.blockfile.cache.BlockCacheManager; +import org.apache.accumulo.core.file.blockfile.cache.BlockCacheManager.Configuration; +import org.apache.accumulo.core.file.blockfile.cache.CacheType; + +public class BlockCacheConfiguration implements Configuration { + + /** Approximate block size */ + private final long blockSize; + + private final Map<String,String> genProps; + + private final long indexMaxSize; + + private final long dataMaxSize; + + private final long summaryMaxSize; + + public BlockCacheConfiguration(AccumuloConfiguration conf) { + genProps = conf.getAllPropertiesWithPrefix(Property.GENERAL_ARBITRARY_PROP_PREFIX); + + this.indexMaxSize = conf.getAsBytes(Property.TSERV_INDEXCACHE_SIZE); + this.dataMaxSize = conf.getAsBytes(Property.TSERV_DATACACHE_SIZE); + this.summaryMaxSize = conf.getAsBytes(Property.TSERV_SUMMARYCACHE_SIZE); + this.blockSize = conf.getAsBytes(Property.TSERV_DEFAULT_BLOCKSIZE); + } + + @Override + public long getMaxSize(CacheType type) { + switch (type) { + case INDEX: + return indexMaxSize; + case DATA: + return dataMaxSize; + case SUMMARY: + return summaryMaxSize; + default: + throw new IllegalArgumentException("Unknown block cache type"); + } + } + + @Override + public long getBlockSize() { + return this.blockSize; + } + + @Override + public String toString() { + return "indexMaxSize: " + indexMaxSize + "dataMaxSize: " + dataMaxSize + "summaryMaxSize: " + summaryMaxSize + ", blockSize: " + getBlockSize(); + } + + @Override + public Map<String,String> getProperties(String prefix, CacheType type) { + HashMap<String,String> props = new HashMap<>(); + + // get default props first + String defaultPrefix = BlockCacheManager.getFullyQualifiedPropertyPrefix(prefix); + genProps.forEach((k, v) -> { + if (k.startsWith(defaultPrefix)) { + props.put(k.substring(defaultPrefix.length()), v); + } + }); + + String typePrefix = BlockCacheManager.getFullyQualifiedPropertyPrefix(prefix, type); + genProps.forEach((k, v) -> { + if (k.startsWith(typePrefix)) { + props.put(k.substring(typePrefix.length()), v); + } + }); + + return Collections.unmodifiableMap(props); + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/d877a2df/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/impl/BlockCacheManagerFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/impl/BlockCacheManagerFactory.java b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/impl/BlockCacheManagerFactory.java new file mode 100644 index 0000000..d78d59c --- /dev/null +++ b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/impl/BlockCacheManagerFactory.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.accumulo.core.file.blockfile.cache.impl; + +import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.file.blockfile.cache.BlockCacheManager; +import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader; + +public class BlockCacheManagerFactory { + + /** + * Get the BlockCacheFactory specified by the property 'tserver.cache.factory.class' using the AccumuloVFSClassLoader + * + * @param conf + * accumulo configuration + * @return block cache manager instance + * @throws Exception + * error loading block cache manager implementation class + */ + public static synchronized BlockCacheManager getInstance(AccumuloConfiguration conf) throws Exception { + String impl = conf.get(Property.TSERV_CACHE_MANAGER_IMPL); + Class<? extends BlockCacheManager> clazz = AccumuloVFSClassLoader.loadClass(impl, BlockCacheManager.class); + BlockCacheManager.LOG.info("Created new block cache manager of type: {}", clazz.getSimpleName()); + return clazz.newInstance(); + } + + /** + * Get the BlockCacheFactory specified by the property 'tserver.cache.factory.class' + * + * @param conf + * accumulo configuration + * @return block cache manager instance + * @throws Exception + * error loading block cache manager implementation class + */ + public static synchronized BlockCacheManager getClientInstance(AccumuloConfiguration conf) throws Exception { + String impl = conf.get(Property.TSERV_CACHE_MANAGER_IMPL); + Class<? extends BlockCacheManager> clazz = Class.forName(impl).asSubclass(BlockCacheManager.class); + BlockCacheManager.LOG.info("Created new block cache factory of type: {}", clazz.getSimpleName()); + return clazz.newInstance(); + } + +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/d877a2df/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/impl/ClassSize.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/impl/ClassSize.java b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/impl/ClassSize.java new file mode 100644 index 0000000..ba540b1 --- /dev/null +++ b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/impl/ClassSize.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.accumulo.core.file.blockfile.cache.impl; + +import java.util.Properties; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * Class for determining the "size" of a class, an attempt to calculate the actual bytes that an object of this class will occupy in memory + * + * The core of this class is taken from the Derby project + */ +public class ClassSize { + static final Log LOG = LogFactory.getLog(ClassSize.class); + + /** Array overhead */ + public static final int ARRAY; + + /** Overhead for ArrayList(0) */ + public static final int ARRAYLIST; + + /** Overhead for ByteBuffer */ + public static final int BYTE_BUFFER; + + /** Overhead for an Integer */ + public static final int INTEGER; + + /** Overhead for entry in map */ + public static final int MAP_ENTRY; + + /** Object overhead is minimum 2 * reference size (8 bytes on 64-bit) */ + public static final int OBJECT; + + /** Reference size is 8 bytes on 64-bit, 4 bytes on 32-bit */ + public static final int REFERENCE; + + /** String overhead */ + public static final int STRING; + + /** Overhead for TreeMap */ + public static final int TREEMAP; + + /** Overhead for ConcurrentHashMap */ + public static final int CONCURRENT_HASHMAP; + + /** Overhead for ConcurrentHashMap.Entry */ + public static final int CONCURRENT_HASHMAP_ENTRY; + + /** Overhead for ConcurrentHashMap.Segment */ + public static final int CONCURRENT_HASHMAP_SEGMENT; + + /** Overhead for ConcurrentSkipListMap */ + public static final int CONCURRENT_SKIPLISTMAP; + + /** Overhead for ConcurrentSkipListMap Entry */ + public static final int CONCURRENT_SKIPLISTMAP_ENTRY; + + /** Overhead for ReentrantReadWriteLock */ + public static final int REENTRANT_LOCK; + + /** Overhead for AtomicLong */ + public static final int ATOMIC_LONG; + + /** Overhead for AtomicInteger */ + public static final int ATOMIC_INTEGER; + + /** Overhead for AtomicBoolean */ + public static final int ATOMIC_BOOLEAN; + + /** Overhead for CopyOnWriteArraySet */ + public static final int COPYONWRITE_ARRAYSET; + + /** Overhead for CopyOnWriteArrayList */ + public static final int COPYONWRITE_ARRAYLIST; + + private static final String THIRTY_TWO = "32"; + + /** + * Method for reading the arc settings and setting overheads according to 32-bit or 64-bit architecture. + */ + static { + // Figure out whether this is a 32 or 64 bit machine. + Properties sysProps = System.getProperties(); + String arcModel = sysProps.getProperty("sun.arch.data.model"); + + // Default value is set to 8, covering the case when arcModel is unknown + REFERENCE = arcModel.equals(THIRTY_TWO) ? 4 : 8; + + OBJECT = 2 * REFERENCE; + + ARRAY = 3 * REFERENCE; + + ARRAYLIST = align(OBJECT + align(REFERENCE) + align(ARRAY) + (2 * SizeConstants.SIZEOF_INT)); + + BYTE_BUFFER = align(OBJECT + align(REFERENCE) + align(ARRAY) + (5 * SizeConstants.SIZEOF_INT) + (3 * SizeConstants.SIZEOF_BOOLEAN) + + SizeConstants.SIZEOF_LONG); + + INTEGER = align(OBJECT + SizeConstants.SIZEOF_INT); + + MAP_ENTRY = align(OBJECT + 5 * REFERENCE + SizeConstants.SIZEOF_BOOLEAN); + + TREEMAP = align(OBJECT + (2 * SizeConstants.SIZEOF_INT) + align(7 * REFERENCE)); + + STRING = align(OBJECT + ARRAY + REFERENCE + 3 * SizeConstants.SIZEOF_INT); + + CONCURRENT_HASHMAP = align((2 * SizeConstants.SIZEOF_INT) + ARRAY + (6 * REFERENCE) + OBJECT); + + CONCURRENT_HASHMAP_ENTRY = align(REFERENCE + OBJECT + (3 * REFERENCE) + (2 * SizeConstants.SIZEOF_INT)); + + CONCURRENT_HASHMAP_SEGMENT = align(REFERENCE + OBJECT + (3 * SizeConstants.SIZEOF_INT) + SizeConstants.SIZEOF_FLOAT + ARRAY); + + CONCURRENT_SKIPLISTMAP = align(SizeConstants.SIZEOF_INT + OBJECT + (8 * REFERENCE)); + + CONCURRENT_SKIPLISTMAP_ENTRY = align(align(OBJECT + (3 * REFERENCE)) + /* one node per entry */ + align((OBJECT + (3 * REFERENCE)) / 2)); /* one index per two entries */ + + REENTRANT_LOCK = align(OBJECT + (3 * REFERENCE)); + + ATOMIC_LONG = align(OBJECT + SizeConstants.SIZEOF_LONG); + + ATOMIC_INTEGER = align(OBJECT + SizeConstants.SIZEOF_INT); + + ATOMIC_BOOLEAN = align(OBJECT + SizeConstants.SIZEOF_BOOLEAN); + + COPYONWRITE_ARRAYSET = align(OBJECT + REFERENCE); + + COPYONWRITE_ARRAYLIST = align(OBJECT + (2 * REFERENCE) + ARRAY); + } + + /** + * Aligns a number to 8. + * + * @param num + * number to align to 8 + * @return smallest number >= input that is a multiple of 8 + */ + public static int align(int num) { + return (int) (align((long) num)); + } + + /** + * Aligns a number to 8. + * + * @param num + * number to align to 8 + * @return smallest number >= input that is a multiple of 8 + */ + public static long align(long num) { + // The 7 comes from that the alignSize is 8 which is the number of bytes + // stored and sent together + return ((num + 7) >> 3) << 3; + } + +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/d877a2df/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/impl/SizeConstants.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/impl/SizeConstants.java b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/impl/SizeConstants.java new file mode 100644 index 0000000..aef769d --- /dev/null +++ b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/impl/SizeConstants.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.file.blockfile.cache.impl; + +public class SizeConstants { + + public static final int SIZEOF_BOOLEAN = Byte.SIZE / Byte.SIZE; + + /** + * Size of byte in bytes + */ + public static final int SIZEOF_BYTE = SIZEOF_BOOLEAN; + + /** + * Size of char in bytes + */ + public static final int SIZEOF_CHAR = Character.SIZE / Byte.SIZE; + + /** + * Size of double in bytes + */ + public static final int SIZEOF_DOUBLE = Double.SIZE / Byte.SIZE; + + /** + * Size of float in bytes + */ + public static final int SIZEOF_FLOAT = Float.SIZE / Byte.SIZE; + + /** + * Size of int in bytes + */ + public static final int SIZEOF_INT = Integer.SIZE / Byte.SIZE; + + /** + * Size of long in bytes + */ + public static final int SIZEOF_LONG = Long.SIZE / Byte.SIZE; + + /** + * Size of short in bytes + */ + public static final int SIZEOF_SHORT = Short.SIZE / Byte.SIZE; + +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/d877a2df/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/CachedBlock.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/CachedBlock.java b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/CachedBlock.java new file mode 100644 index 0000000..c1124f4 --- /dev/null +++ b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/CachedBlock.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.file.blockfile.cache.lru; + +import java.util.Objects; + +import org.apache.accumulo.core.file.blockfile.cache.CacheEntry; +import org.apache.accumulo.core.file.blockfile.cache.impl.ClassSize; +import org.apache.accumulo.core.file.blockfile.cache.impl.SizeConstants; + +/** + * Represents an entry in the configurable block cache. + * + * <p> + * Makes the block memory-aware with {@link HeapSize} and Comparable to sort by access time for the LRU. It also takes care of priority by either instantiating + * as in-memory or handling the transition from single to multiple access. + */ +public class CachedBlock implements HeapSize, Comparable<CachedBlock>, CacheEntry { + + public final static long PER_BLOCK_OVERHEAD = ClassSize.align(ClassSize.OBJECT + (3 * ClassSize.REFERENCE) + (2 * SizeConstants.SIZEOF_LONG) + + ClassSize.STRING + ClassSize.BYTE_BUFFER); + + public static enum BlockPriority { + /** + * Accessed a single time (used for scan-resistance) + */ + SINGLE, + /** + * Accessed multiple times + */ + MULTI, + /** + * Block from in-memory store + */ + MEMORY + } + + private final String blockName; + private final byte buf[]; + private volatile long accessTime; + private long size; + private BlockPriority priority; + private Object index; + + public CachedBlock(String blockName, byte buf[], long accessTime, boolean inMemory) { + this.blockName = blockName; + this.buf = buf; + this.accessTime = accessTime; + this.size = ClassSize.align(blockName.length()) + ClassSize.align(buf.length) + PER_BLOCK_OVERHEAD; + if (inMemory) { + this.priority = BlockPriority.MEMORY; + } else { + this.priority = BlockPriority.SINGLE; + } + } + + /** + * Block has been accessed. Update its local access time. + */ + public void access(long accessTime) { + this.accessTime = accessTime; + if (this.priority == BlockPriority.SINGLE) { + this.priority = BlockPriority.MULTI; + } + } + + @Override + public long heapSize() { + return size; + } + + @Override + public int hashCode() { + return Objects.hashCode(accessTime); + } + + @Override + public boolean equals(Object obj) { + return this == obj || (obj != null && obj instanceof CachedBlock && 0 == compareTo((CachedBlock) obj)); + } + + @Override + public int compareTo(CachedBlock that) { + if (this.accessTime == that.accessTime) + return 0; + return this.accessTime < that.accessTime ? 1 : -1; + } + + @Override + public byte[] getBuffer() { + return this.buf; + } + + public String getName() { + return this.blockName; + } + + public BlockPriority getPriority() { + return this.priority; + } + + @Override + public Object getIndex() { + return index; + } + + @Override + public void setIndex(Object idx) { + this.index = idx; + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/d877a2df/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/CachedBlockQueue.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/CachedBlockQueue.java b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/CachedBlockQueue.java new file mode 100644 index 0000000..3906c5e --- /dev/null +++ b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/CachedBlockQueue.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.file.blockfile.cache.lru; + +import java.util.LinkedList; +import java.util.PriorityQueue; + +/** + * A memory-bound queue that will grow until an element brings total size >= maxSize. From then on, only entries that are sorted larger than the smallest + * current entry will be inserted/replaced. + * + * <p> + * Use this when you want to find the largest elements (according to their ordering, not their heap size) that consume as close to the specified maxSize as + * possible. Default behavior is to grow just above rather than just below specified max. + * + * <p> + * Object used in this queue must implement {@link HeapSize} as well as {@link Comparable}. + */ +public class CachedBlockQueue implements HeapSize { + + private PriorityQueue<CachedBlock> queue; + + private long heapSize; + private long maxSize; + + /** + * @param maxSize + * the target size of elements in the queue + * @param blockSize + * expected average size of blocks + */ + public CachedBlockQueue(long maxSize, long blockSize) { + int initialSize = (int) Math.ceil(maxSize / (double) blockSize); + if (initialSize == 0) + initialSize++; + queue = new PriorityQueue<>(initialSize); + heapSize = 0; + this.maxSize = maxSize; + } + + /** + * Attempt to add the specified cached block to this queue. + * + * <p> + * If the queue is smaller than the max size, or if the specified element is ordered before the smallest element in the queue, the element will be added to + * the queue. Otherwise, there is no side effect of this call. + * + * @param cb + * block to try to add to the queue + */ + public void add(CachedBlock cb) { + if (heapSize < maxSize) { + queue.add(cb); + heapSize += cb.heapSize(); + } else { + CachedBlock head = queue.peek(); + if (cb.compareTo(head) > 0) { + heapSize += cb.heapSize(); + heapSize -= head.heapSize(); + if (heapSize > maxSize) { + queue.poll(); + } else { + heapSize += head.heapSize(); + } + queue.add(cb); + } + } + } + + /** + * Get a sorted List of all elements in this queue, in descending order. + * + * @return list of cached elements in descending order + */ + public CachedBlock[] get() { + LinkedList<CachedBlock> blocks = new LinkedList<>(); + while (!queue.isEmpty()) { + blocks.addFirst(queue.poll()); + } + return blocks.toArray(new CachedBlock[blocks.size()]); + } + + /** + * Get a sorted List of all elements in this queue, in descending order. + * + * @return list of cached elements in descending order + */ + public LinkedList<CachedBlock> getList() { + LinkedList<CachedBlock> blocks = new LinkedList<>(); + while (!queue.isEmpty()) { + blocks.addFirst(queue.poll()); + } + return blocks; + } + + /** + * Total size of all elements in this queue. + * + * @return size of all elements currently in queue, in bytes + */ + @Override + public long heapSize() { + return heapSize; + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/d877a2df/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/HeapSize.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/HeapSize.java b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/HeapSize.java new file mode 100644 index 0000000..1809170 --- /dev/null +++ b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/HeapSize.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.file.blockfile.cache.lru; + +/** + * Implementations can be asked for an estimate of their size in bytes. + * <p> + * Useful for sizing caches. Its a given that implementation approximations do not account for 32 vs 64 bit nor for different VM implementations. + * <p> + * An Object's size is determined by the non-static data members in it, as well as the fixed {@link Object} overhead. + * <p> + * For example: + * + * <pre> + * public class SampleObject implements HeapSize { + * int[] numbers; + * int x; + * } + * </pre> + */ +public interface HeapSize { + /** + * @return Approximate 'exclusive deep size' of implementing object. Includes count of payload and hosting object sizings. + */ + long heapSize(); + +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/d877a2df/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCache.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCache.java b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCache.java index fa8d824..f27fd43 100644 --- a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCache.java +++ b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCache.java @@ -29,11 +29,8 @@ import java.util.concurrent.locks.ReentrantLock; import org.apache.accumulo.core.file.blockfile.cache.BlockCache; import org.apache.accumulo.core.file.blockfile.cache.CacheEntry; -import org.apache.accumulo.core.file.blockfile.cache.CachedBlock; -import org.apache.accumulo.core.file.blockfile.cache.CachedBlockQueue; -import org.apache.accumulo.core.file.blockfile.cache.ClassSize; -import org.apache.accumulo.core.file.blockfile.cache.HeapSize; -import org.apache.accumulo.core.file.blockfile.cache.SizeConstants; +import org.apache.accumulo.core.file.blockfile.cache.impl.ClassSize; +import org.apache.accumulo.core.file.blockfile.cache.impl.SizeConstants; import org.apache.accumulo.core.util.NamingThreadFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/accumulo/blob/d877a2df/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCacheConfiguration.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCacheConfiguration.java b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCacheConfiguration.java index 49790cb..b82c24b 100644 --- a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCacheConfiguration.java +++ b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCacheConfiguration.java @@ -19,15 +19,16 @@ package org.apache.accumulo.core.file.blockfile.cache.lru; import java.util.HashMap; import java.util.Map; +import java.util.Optional; -import org.apache.accumulo.core.conf.AccumuloConfiguration; -import org.apache.accumulo.core.file.blockfile.cache.BlockCacheConfiguration; +import org.apache.accumulo.core.file.blockfile.cache.BlockCacheManager; +import org.apache.accumulo.core.file.blockfile.cache.BlockCacheManager.Configuration; import org.apache.accumulo.core.file.blockfile.cache.CacheType; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; -public final class LruBlockCacheConfiguration extends BlockCacheConfiguration { +public final class LruBlockCacheConfiguration { public static final String PROPERTY_PREFIX = "lru"; @@ -79,8 +80,21 @@ public final class LruBlockCacheConfiguration extends BlockCacheConfiguration { private final boolean useEvictionThread; - public LruBlockCacheConfiguration(AccumuloConfiguration conf, CacheType type) { - super(conf, type, PROPERTY_PREFIX); + private final Configuration conf; + + private final Map<String,String> props; + + private final CacheType type; + + private Optional<String> get(String k) { + return Optional.ofNullable(props.get(k)); + } + + public LruBlockCacheConfiguration(Configuration conf, CacheType type) { + + this.type = type; + this.conf = conf; + this.props = conf.getProperties(PROPERTY_PREFIX, type); this.acceptableFactor = get(ACCEPTABLE_FACTOR_PROPERTY).map(Float::valueOf).filter(f -> f > 0).orElse(DEFAULT_ACCEPTABLE_FACTOR); this.minFactor = get(MIN_FACTOR_PROPERTY).map(Float::valueOf).filter(f -> f > 0).orElse(DEFAULT_MIN_FACTOR); @@ -199,7 +213,7 @@ public final class LruBlockCacheConfiguration extends BlockCacheConfiguration { } public static Builder builder(CacheType ct) { - return new Builder(getPrefix(ct, PROPERTY_PREFIX)); + return new Builder(BlockCacheManager.getFullyQualifiedPropertyPrefix(PROPERTY_PREFIX, ct)); } @Override @@ -209,4 +223,12 @@ public final class LruBlockCacheConfiguration extends BlockCacheConfiguration { + this.getMapLoadFactor() + ", mapConcurrencyLevel: " + this.getMapConcurrencyLevel() + ", useEvictionThread: " + this.isUseEvictionThread(); } + public long getMaxSize() { + return conf.getMaxSize(type); + } + + public long getBlockSize() { + return conf.getBlockSize(); + } + } http://git-wip-us.apache.org/repos/asf/accumulo/blob/d877a2df/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCacheManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCacheManager.java b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCacheManager.java index 8a1e430..f055e98 100644 --- a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCacheManager.java +++ b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCacheManager.java @@ -17,7 +17,6 @@ */ package org.apache.accumulo.core.file.blockfile.cache.lru; -import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.file.blockfile.cache.BlockCache; import org.apache.accumulo.core.file.blockfile.cache.BlockCacheManager; import org.apache.accumulo.core.file.blockfile.cache.CacheType; @@ -29,7 +28,7 @@ public class LruBlockCacheManager extends BlockCacheManager { private static final Logger LOG = LoggerFactory.getLogger(LruBlockCacheManager.class); @Override - protected BlockCache createCache(AccumuloConfiguration conf, CacheType type) { + protected BlockCache createCache(Configuration conf, CacheType type) { LruBlockCacheConfiguration cc = new LruBlockCacheConfiguration(conf, type); LOG.info("Creating {} cache with configuration {}", type, cc); return new LruBlockCache(cc); http://git-wip-us.apache.org/repos/asf/accumulo/blob/d877a2df/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java index db4e789..6edfb01 100644 --- a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java +++ b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java @@ -24,9 +24,11 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import org.apache.accumulo.core.file.blockfile.cache.BlockCache; +import org.apache.accumulo.core.file.blockfile.cache.BlockCacheManager.Configuration; import org.apache.accumulo.core.file.blockfile.cache.CacheEntry; -import org.apache.accumulo.core.file.blockfile.cache.ClassSize; -import org.apache.accumulo.core.file.blockfile.cache.SizeConstants; +import org.apache.accumulo.core.file.blockfile.cache.CacheType; +import org.apache.accumulo.core.file.blockfile.cache.impl.ClassSize; +import org.apache.accumulo.core.file.blockfile.cache.impl.SizeConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,11 +55,12 @@ public final class TinyLfuBlockCache implements BlockCache { private Policy.Eviction<String,Block> policy; private ScheduledExecutorService statsExecutor; - public TinyLfuBlockCache(TinyLfuBlockCacheConfiguration conf) { - cache = Caffeine.newBuilder().initialCapacity((int) Math.ceil(1.2 * conf.getMaxSize() / conf.getBlockSize())).weigher((String blockName, Block block) -> { - int keyWeight = ClassSize.align(blockName.length()) + ClassSize.STRING; - return keyWeight + block.weight(); - }).maximumWeight(conf.getMaxSize()).recordStats().build(); + public TinyLfuBlockCache(Configuration conf, CacheType type) { + cache = Caffeine.newBuilder().initialCapacity((int) Math.ceil(1.2 * conf.getMaxSize(type) / conf.getBlockSize())) + .weigher((String blockName, Block block) -> { + int keyWeight = ClassSize.align(blockName.length()) + ClassSize.STRING; + return keyWeight + block.weight(); + }).maximumWeight(conf.getMaxSize(type)).recordStats().build(); policy = cache.policy().eviction().get(); statsExecutor = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryBuilder().setNameFormat("TinyLfuBlockCacheStatsExecutor").setDaemon(true) .build()); http://git-wip-us.apache.org/repos/asf/accumulo/blob/d877a2df/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCacheConfiguration.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCacheConfiguration.java b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCacheConfiguration.java deleted file mode 100644 index a67f164..0000000 --- a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCacheConfiguration.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.core.file.blockfile.cache.tinylfu; - -import org.apache.accumulo.core.conf.AccumuloConfiguration; -import org.apache.accumulo.core.file.blockfile.cache.BlockCacheConfiguration; -import org.apache.accumulo.core.file.blockfile.cache.CacheType; - -public final class TinyLfuBlockCacheConfiguration extends BlockCacheConfiguration { - - public static final String PROPERTY_PREFIX = "tinylfu"; - - public TinyLfuBlockCacheConfiguration(AccumuloConfiguration conf, CacheType type) { - super(conf, type, PROPERTY_PREFIX); - } - -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/d877a2df/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCacheManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCacheManager.java b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCacheManager.java index a68c4e6..e7369e0 100644 --- a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCacheManager.java +++ b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCacheManager.java @@ -17,7 +17,6 @@ */ package org.apache.accumulo.core.file.blockfile.cache.tinylfu; -import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.file.blockfile.cache.BlockCacheManager; import org.apache.accumulo.core.file.blockfile.cache.CacheType; import org.slf4j.Logger; @@ -28,10 +27,9 @@ public class TinyLfuBlockCacheManager extends BlockCacheManager { private static final Logger LOG = LoggerFactory.getLogger(TinyLfuBlockCacheManager.class); @Override - protected TinyLfuBlockCache createCache(AccumuloConfiguration conf, CacheType type) { - TinyLfuBlockCacheConfiguration cc = new TinyLfuBlockCacheConfiguration(conf, type); - LOG.info("Creating {} cache with configuration {}", type, cc); - return new TinyLfuBlockCache(cc); + protected TinyLfuBlockCache createCache(Configuration conf, CacheType type) { + LOG.info("Creating {} cache with configuration {}", type, conf); + return new TinyLfuBlockCache(conf, type); } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/d877a2df/core/src/test/java/org/apache/accumulo/core/file/blockfile/cache/BlockCacheFactoryTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/file/blockfile/cache/BlockCacheFactoryTest.java b/core/src/test/java/org/apache/accumulo/core/file/blockfile/cache/BlockCacheFactoryTest.java index e17fb76..b746003 100644 --- a/core/src/test/java/org/apache/accumulo/core/file/blockfile/cache/BlockCacheFactoryTest.java +++ b/core/src/test/java/org/apache/accumulo/core/file/blockfile/cache/BlockCacheFactoryTest.java @@ -20,6 +20,8 @@ package org.apache.accumulo.core.file.blockfile.cache; import org.apache.accumulo.core.conf.ConfigurationCopy; import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.file.blockfile.cache.impl.BlockCacheConfiguration; +import org.apache.accumulo.core.file.blockfile.cache.impl.BlockCacheManagerFactory; import org.apache.accumulo.core.file.blockfile.cache.lru.LruBlockCacheManager; import org.apache.accumulo.core.file.blockfile.cache.tinylfu.TinyLfuBlockCacheManager; import org.junit.Assert; @@ -32,7 +34,7 @@ public class BlockCacheFactoryTest { DefaultConfiguration dc = DefaultConfiguration.getInstance(); ConfigurationCopy cc = new ConfigurationCopy(dc); cc.set(Property.TSERV_CACHE_MANAGER_IMPL, LruBlockCacheManager.class.getName()); - BlockCacheManager.getInstance(cc); + BlockCacheManagerFactory.getInstance(cc); } @Test @@ -40,14 +42,14 @@ public class BlockCacheFactoryTest { DefaultConfiguration dc = DefaultConfiguration.getInstance(); ConfigurationCopy cc = new ConfigurationCopy(dc); cc.set(Property.TSERV_CACHE_MANAGER_IMPL, TinyLfuBlockCacheManager.class.getName()); - BlockCacheManager.getInstance(cc); + BlockCacheManagerFactory.getInstance(cc); } @Test public void testStartWithDefault() throws Exception { DefaultConfiguration dc = DefaultConfiguration.getInstance(); - BlockCacheManager manager = BlockCacheManager.getInstance(dc); - manager.start(dc); + BlockCacheManager manager = BlockCacheManagerFactory.getInstance(dc); + manager.start(new BlockCacheConfiguration(dc)); Assert.assertNotNull(manager.getBlockCache(CacheType.INDEX)); } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/d877a2df/core/src/test/java/org/apache/accumulo/core/file/blockfile/cache/BlockCacheManagerTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/file/blockfile/cache/BlockCacheManagerTest.java b/core/src/test/java/org/apache/accumulo/core/file/blockfile/cache/BlockCacheManagerTest.java new file mode 100644 index 0000000..7bfd511 --- /dev/null +++ b/core/src/test/java/org/apache/accumulo/core/file/blockfile/cache/BlockCacheManagerTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.file.blockfile.cache; + +import org.junit.Assert; +import org.junit.Test; + +public class BlockCacheManagerTest { + + @Test + public void testGetPropertyPrefix() throws Exception { + Assert.assertEquals("general.custom.cache.lru.data.", BlockCacheManager.getFullyQualifiedPropertyPrefix("lru", CacheType.DATA)); + Assert.assertEquals("general.custom.cache.lru.index.", BlockCacheManager.getFullyQualifiedPropertyPrefix("lru", CacheType.INDEX)); + Assert.assertEquals("general.custom.cache.lru.summary.", BlockCacheManager.getFullyQualifiedPropertyPrefix("lru", CacheType.SUMMARY)); + Assert.assertEquals("general.custom.cache.lru.default.", BlockCacheManager.getFullyQualifiedPropertyPrefix("lru")); + } + +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/d877a2df/core/src/test/java/org/apache/accumulo/core/file/blockfile/cache/BlockConfigurationHelperTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/file/blockfile/cache/BlockConfigurationHelperTest.java b/core/src/test/java/org/apache/accumulo/core/file/blockfile/cache/BlockConfigurationHelperTest.java deleted file mode 100644 index 72ea49c..0000000 --- a/core/src/test/java/org/apache/accumulo/core/file/blockfile/cache/BlockConfigurationHelperTest.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.core.file.blockfile.cache; - -import org.junit.Assert; -import org.junit.Test; - -public class BlockConfigurationHelperTest { - - @Test - public void testGetPropertyPrefix() throws Exception { - Assert.assertEquals("general.custom.cache.block.lru.data.", BlockCacheConfiguration.getPrefix(CacheType.DATA, "lru")); - Assert.assertEquals("general.custom.cache.block.lru.index.", BlockCacheConfiguration.getPrefix(CacheType.INDEX, "lru")); - Assert.assertEquals("general.custom.cache.block.lru.summary.", BlockCacheConfiguration.getPrefix(CacheType.SUMMARY, "lru")); - } - -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/d877a2df/core/src/test/java/org/apache/accumulo/core/file/blockfile/cache/TestCachedBlockQueue.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/file/blockfile/cache/TestCachedBlockQueue.java b/core/src/test/java/org/apache/accumulo/core/file/blockfile/cache/TestCachedBlockQueue.java index e88b4a9..8d45124 100644 --- a/core/src/test/java/org/apache/accumulo/core/file/blockfile/cache/TestCachedBlockQueue.java +++ b/core/src/test/java/org/apache/accumulo/core/file/blockfile/cache/TestCachedBlockQueue.java @@ -19,6 +19,8 @@ package org.apache.accumulo.core.file.blockfile.cache; import java.util.LinkedList; +import org.apache.accumulo.core.file.blockfile.cache.lru.CachedBlockQueue; + import junit.framework.TestCase; public class TestCachedBlockQueue extends TestCase { @@ -54,7 +56,7 @@ public class TestCachedBlockQueue extends TestCase { assertEquals(queue.heapSize(), expectedSize); - LinkedList<org.apache.accumulo.core.file.blockfile.cache.CachedBlock> blocks = queue.getList(); + LinkedList<org.apache.accumulo.core.file.blockfile.cache.lru.CachedBlock> blocks = queue.getList(); assertEquals(blocks.poll().getName(), "cb1"); assertEquals(blocks.poll().getName(), "cb2"); assertEquals(blocks.poll().getName(), "cb3"); @@ -105,7 +107,7 @@ public class TestCachedBlockQueue extends TestCase { assertEquals(queue.heapSize(), expectedSize); - LinkedList<org.apache.accumulo.core.file.blockfile.cache.CachedBlock> blocks = queue.getList(); + LinkedList<org.apache.accumulo.core.file.blockfile.cache.lru.CachedBlock> blocks = queue.getList(); assertEquals(blocks.poll().getName(), "cb0"); assertEquals(blocks.poll().getName(), "cb1"); assertEquals(blocks.poll().getName(), "cb2"); @@ -118,7 +120,7 @@ public class TestCachedBlockQueue extends TestCase { } - private static class CachedBlock extends org.apache.accumulo.core.file.blockfile.cache.CachedBlock { + private static class CachedBlock extends org.apache.accumulo.core.file.blockfile.cache.lru.CachedBlock { public CachedBlock(long heapSize, String name, long accessTime) { super(name, new byte[(int) (heapSize - CachedBlock.PER_BLOCK_OVERHEAD)], accessTime, false); }
