Renamed occurences of bank to slab in BlockCache
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/172a058a Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/172a058a Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/172a058a Branch: refs/heads/master Commit: 172a058a9c4ff46205ce86a7a518a73fbbe08f5d Parents: d4a4353 Author: Scott Leberknight <[email protected]> Authored: Mon Aug 27 18:04:46 2012 -0400 Committer: Scott Leberknight <[email protected]> Committed: Mon Aug 27 18:04:46 2012 -0400 ---------------------------------------------------------------------- .../blur/thrift/ThriftBlurShardServer.java | 62 +++++++------- .../blur/store/blockcache/BlockCache.java | 64 +++++++------- .../blur/store/blockcache/BlockCacheLocation.java | 10 +- .../blur/store/BenchmarkDirectory.java | 8 +- .../blur/store/BenchmarkDirectoryNrt.java | 8 +- 5 files changed, 76 insertions(+), 76 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/172a058a/src/blur-core/src/main/java/com/nearinfinity/blur/thrift/ThriftBlurShardServer.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/com/nearinfinity/blur/thrift/ThriftBlurShardServer.java b/src/blur-core/src/main/java/com/nearinfinity/blur/thrift/ThriftBlurShardServer.java index bb1243d..6b1d07f 100644 --- a/src/blur-core/src/main/java/com/nearinfinity/blur/thrift/ThriftBlurShardServer.java +++ b/src/blur-core/src/main/java/com/nearinfinity/blur/thrift/ThriftBlurShardServer.java @@ -16,28 +16,6 @@ package com.nearinfinity.blur.thrift; -import static com.nearinfinity.blur.utils.BlurConstants.BLUR_CLUSTER_NAME; -import static com.nearinfinity.blur.utils.BlurConstants.BLUR_INDEXMANAGER_SEARCH_THREAD_COUNT; -import static com.nearinfinity.blur.utils.BlurConstants.BLUR_MAX_CLAUSE_COUNT; -import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_BIND_ADDRESS; -import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_BIND_PORT; -import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_BLOCKCACHE_DIRECT_MEMORY_ALLOCATION; -import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_BLOCKCACHE_SLAB_COUNT; -import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_CACHE_MAX_QUERYCACHE_ELEMENTS; -import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_CACHE_MAX_TIMETOLIVE; -import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_DATA_FETCH_THREAD_COUNT; -import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_FILTER_CACHE_CLASS; -import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_HOSTNAME; -import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_INDEX_WARMUP_CLASS; -import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_OPENER_THREAD_COUNT; -import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_SAFEMODEDELAY; -import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_SERVER_THRIFT_THREAD_COUNT; -import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_TIME_BETWEEN_COMMITS; -import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_TIME_BETWEEN_REFRESHS; -import static com.nearinfinity.blur.utils.BlurConstants.BLUR_ZOOKEEPER_CONNECTION; -import static com.nearinfinity.blur.utils.BlurConstants.BLUR_ZOOKEEPER_SYSTEM_TIME_TOLERANCE; -import static com.nearinfinity.blur.utils.BlurUtil.quietClose; - import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; @@ -73,6 +51,28 @@ import com.nearinfinity.blur.thrift.generated.Blur.Iface; import com.nearinfinity.blur.utils.BlurUtil; import com.nearinfinity.blur.zookeeper.ZkUtils; +import static com.nearinfinity.blur.utils.BlurConstants.BLUR_CLUSTER_NAME; +import static com.nearinfinity.blur.utils.BlurConstants.BLUR_INDEXMANAGER_SEARCH_THREAD_COUNT; +import static com.nearinfinity.blur.utils.BlurConstants.BLUR_MAX_CLAUSE_COUNT; +import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_BIND_ADDRESS; +import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_BIND_PORT; +import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_BLOCKCACHE_DIRECT_MEMORY_ALLOCATION; +import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_BLOCKCACHE_SLAB_COUNT; +import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_CACHE_MAX_QUERYCACHE_ELEMENTS; +import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_CACHE_MAX_TIMETOLIVE; +import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_DATA_FETCH_THREAD_COUNT; +import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_FILTER_CACHE_CLASS; +import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_HOSTNAME; +import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_INDEX_WARMUP_CLASS; +import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_OPENER_THREAD_COUNT; +import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_SAFEMODEDELAY; +import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_SERVER_THRIFT_THREAD_COUNT; +import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_TIME_BETWEEN_COMMITS; +import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_TIME_BETWEEN_REFRESHS; +import static com.nearinfinity.blur.utils.BlurConstants.BLUR_ZOOKEEPER_CONNECTION; +import static com.nearinfinity.blur.utils.BlurConstants.BLUR_ZOOKEEPER_SYSTEM_TIME_TOLERANCE; +import static com.nearinfinity.blur.utils.BlurUtil.quietClose; + public class ThriftBlurShardServer extends ThriftServer { private static final Log LOG = LogFactory.getLog(ThriftBlurShardServer.class); @@ -90,26 +90,26 @@ public class ThriftBlurShardServer extends ThriftServer { public static ThriftServer createServer(int serverIndex, BlurConfiguration configuration) throws Exception { // setup block cache - // 134,217,728 is the bank size, therefore there are 16,384 block - // in a bank when using a block of 8,192 - int numberOfBlocksPerBank = 16384; + // 134,217,728 is the slab size, therefore there are 16,384 blocks + // in a slab when using a block size of 8,192 + int numberOfBlocksPerSlab = 16384; int blockSize = BlockDirectory.BLOCK_SIZE; - int bankCount = configuration.getInt(BLUR_SHARD_BLOCKCACHE_SLAB_COUNT, 1); + int slabCount = configuration.getInt(BLUR_SHARD_BLOCKCACHE_SLAB_COUNT, 1); Cache cache; Configuration config = new Configuration(); BlurMetrics blurMetrics = new BlurMetrics(config); - if (bankCount >= 1) { + if (slabCount >= 1) { BlockCache blockCache; boolean directAllocation = configuration.getBoolean(BLUR_SHARD_BLOCKCACHE_DIRECT_MEMORY_ALLOCATION, true); - int slabSize = numberOfBlocksPerBank * blockSize; - LOG.info("Number of slabs of block cache [{0}] with direct memory allocation set to [{1}]", bankCount, directAllocation); - LOG.info("Block cache target memory usage, slab size of [{0}] will allocate [{1}] slabs and use ~[{2}] bytes", slabSize, bankCount, ((long) bankCount * (long) slabSize)); + int slabSize = numberOfBlocksPerSlab * blockSize; + LOG.info("Number of slabs of block cache [{0}] with direct memory allocation set to [{1}]", slabCount, directAllocation); + LOG.info("Block cache target memory usage, slab size of [{0}] will allocate [{1}] slabs and use ~[{2}] bytes", slabSize, slabCount, ((long) slabCount * (long) slabSize)); BufferStore.init(configuration, blurMetrics); try { - long totalMemory = (long) bankCount * (long) numberOfBlocksPerBank * (long) blockSize; + long totalMemory = (long) slabCount * (long) numberOfBlocksPerSlab * (long) blockSize; blockCache = new BlockCache(blurMetrics, directAllocation, totalMemory, slabSize, blockSize); } catch (OutOfMemoryError e) { if ("Direct buffer memory".equals(e.getMessage())) { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/172a058a/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockCache.java ---------------------------------------------------------------------- diff --git a/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockCache.java b/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockCache.java index 349fc34..ecd2960 100644 --- a/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockCache.java +++ b/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockCache.java @@ -13,11 +13,11 @@ public class BlockCache { public static final int _128M = 134217728; public static final int _32K = 32768; private final ConcurrentMap<BlockCacheKey, BlockCacheLocation> _cache; - private final ByteBuffer[] _banks; + private final ByteBuffer[] _slabs; private final BlockLocks[] _locks; private final AtomicInteger[] _lockCounters; private final int _blockSize; - private final int _numberOfBlocksPerBank; + private final int _numberOfBlocksPerSlab; private final int _maxEntries; private final BlurMetrics _metrics; @@ -31,20 +31,20 @@ public class BlockCache { public BlockCache(BlurMetrics metrics, boolean directAllocation, long totalMemory, int slabSize, int blockSize) { _metrics = metrics; - _numberOfBlocksPerBank = slabSize / blockSize; - int numberOfBanks = (int) (totalMemory / slabSize); + _numberOfBlocksPerSlab = slabSize / blockSize; + int numberOfSlabs = (int) (totalMemory / slabSize); - _banks = new ByteBuffer[numberOfBanks]; - _locks = new BlockLocks[numberOfBanks]; - _lockCounters = new AtomicInteger[numberOfBanks]; - _maxEntries = (_numberOfBlocksPerBank * numberOfBanks) - 1; - for (int i = 0; i < numberOfBanks; i++) { + _slabs = new ByteBuffer[numberOfSlabs]; + _locks = new BlockLocks[numberOfSlabs]; + _lockCounters = new AtomicInteger[numberOfSlabs]; + _maxEntries = (_numberOfBlocksPerSlab * numberOfSlabs) - 1; + for (int i = 0; i < numberOfSlabs; i++) { if (directAllocation) { - _banks[i] = ByteBuffer.allocateDirect(_numberOfBlocksPerBank * blockSize); + _slabs[i] = ByteBuffer.allocateDirect(_numberOfBlocksPerSlab * blockSize); } else { - _banks[i] = ByteBuffer.allocate(_numberOfBlocksPerBank * blockSize); + _slabs[i] = ByteBuffer.allocate(_numberOfBlocksPerSlab * blockSize); } - _locks[i] = new BlockLocks(_numberOfBlocksPerBank); + _locks[i] = new BlockLocks(_numberOfBlocksPerSlab); _lockCounters[i] = new AtomicInteger(); } @@ -62,11 +62,11 @@ public class BlockCache { if (location == null) { return; } - int bankId = location.getBankId(); + int slabId = location.getSlabId(); int block = location.getBlock(); location.setRemoved(true); - _locks[bankId].clear(block); - _lockCounters[bankId].decrementAndGet(); + _locks[slabId].clear(block); + _lockCounters[slabId].decrementAndGet(); _metrics.blockCacheEviction.incrementAndGet(); _metrics.blockCacheSize.decrementAndGet(); } @@ -85,11 +85,11 @@ public class BlockCache { if (location.isRemoved()) { return false; } - int bankId = location.getBankId(); + int slabId = location.getSlabId(); int offset = location.getBlock() * _blockSize; - ByteBuffer bank = getBank(bankId); - bank.position(offset); - bank.put(data, 0, _blockSize); + ByteBuffer slab = getSlab(slabId); + slab.position(offset); + slab.put(data, 0, _blockSize); if (newLocation) { releaseLocation(_cache.put(blockCacheKey.clone(), location)); _metrics.blockCacheSize.incrementAndGet(); @@ -105,12 +105,12 @@ public class BlockCache { if (location.isRemoved()) { return false; } - int bankId = location.getBankId(); + int slabId = location.getSlabId(); int offset = location.getBlock() * _blockSize; location.touch(); - ByteBuffer bank = getBank(bankId); - bank.position(offset + blockOffset); - bank.get(buffer, off, length); + ByteBuffer slab = getSlab(slabId); + slab.position(offset + blockOffset); + slab.get(buffer, off, length); return true; } @@ -123,18 +123,18 @@ public class BlockCache { // This is a tight loop that will try and find a location to // place the block before giving up for (int j = 0; j < 10; j++) { - OUTER: for (int bankId = 0; bankId < _banks.length; bankId++) { - AtomicInteger bitSetCounter = _lockCounters[bankId]; - BlockLocks bitSet = _locks[bankId]; - if (bitSetCounter.get() == _numberOfBlocksPerBank) { + OUTER: for (int slabId = 0; slabId < _slabs.length; slabId++) { + AtomicInteger bitSetCounter = _lockCounters[slabId]; + BlockLocks bitSet = _locks[slabId]; + if (bitSetCounter.get() == _numberOfBlocksPerSlab) { // if bitset is full continue OUTER; } // this check needs to spin, if a lock was attempted but not obtained - // the rest of the bank should not be skipped + // the rest of the slab should not be skipped int bit = bitSet.nextClearBit(0); INNER: while (bit != -1) { - if (bit >= _numberOfBlocksPerBank) { + if (bit >= _numberOfBlocksPerSlab) { // bit set is full continue OUTER; } @@ -146,7 +146,7 @@ public class BlockCache { continue INNER; } else { // lock obtained - location.setBankId(bankId); + location.setSlabId(slabId); location.setBlock(bit); bitSetCounter.incrementAndGet(); return true; @@ -163,8 +163,8 @@ public class BlockCache { } } - private ByteBuffer getBank(int bankId) { - return _banks[bankId].duplicate(); + private ByteBuffer getSlab(int slabId) { + return _slabs[slabId].duplicate(); } public int getSize() { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/172a058a/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockCacheLocation.java ---------------------------------------------------------------------- diff --git a/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockCacheLocation.java b/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockCacheLocation.java index 4940ce1..b394894 100644 --- a/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockCacheLocation.java +++ b/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockCacheLocation.java @@ -6,7 +6,7 @@ import java.util.concurrent.atomic.AtomicBoolean; public class BlockCacheLocation { private int _block; - private int _bankId; + private int _slabId; private long _lastAccess = System.currentTimeMillis(); private long _accesses; private AtomicBoolean _removed = new AtomicBoolean(false); @@ -15,16 +15,16 @@ public class BlockCacheLocation { _block = block; } - public void setBankId(int bankId) { - _bankId = bankId; + public void setSlabId(int slabId) { + _slabId = slabId; } public int getBlock() { return _block; } - public int getBankId() { - return _bankId; + public int getSlabId() { + return _slabId; } public void touch() { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/172a058a/src/blur-store/src/test/java/com/nearinfinity/blur/store/BenchmarkDirectory.java ---------------------------------------------------------------------- diff --git a/src/blur-store/src/test/java/com/nearinfinity/blur/store/BenchmarkDirectory.java b/src/blur-store/src/test/java/com/nearinfinity/blur/store/BenchmarkDirectory.java index 91c28f1..93eda49 100644 --- a/src/blur-store/src/test/java/com/nearinfinity/blur/store/BenchmarkDirectory.java +++ b/src/blur-store/src/test/java/com/nearinfinity/blur/store/BenchmarkDirectory.java @@ -1,7 +1,5 @@ package com.nearinfinity.blur.store; -import static com.nearinfinity.blur.lucene.LuceneConstant.LUCENE_VERSION; - import java.io.IOException; import java.lang.management.ManagementFactory; import java.util.ArrayList; @@ -35,6 +33,8 @@ import com.nearinfinity.blur.store.blockcache.BlockDirectoryCache; import com.nearinfinity.blur.store.hdfs.HdfsDirectory; import com.nearinfinity.blur.store.lock.BlurLockFactory; +import static com.nearinfinity.blur.lucene.LuceneConstant.LUCENE_VERSION; + public class BenchmarkDirectory { public static void main(String[] args) throws IOException { @@ -119,10 +119,10 @@ public class BenchmarkDirectory { return document; } - public static int getNumberOfBanks(float heapPercentage, int numberOfBlocksPerBank, int blockSize) { + public static int getNumberOfSlabs(float heapPercentage, int numberOfBlocksPerSlab, int blockSize) { long max = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax(); long targetBytes = (long) (max * heapPercentage); - int slabSize = numberOfBlocksPerBank * blockSize; + int slabSize = numberOfBlocksPerSlab * blockSize; int slabs = (int) (targetBytes / slabSize); if (slabs == 0) { throw new RuntimeException("Minimum heap size is 512m!"); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/172a058a/src/blur-store/src/test/java/com/nearinfinity/blur/store/BenchmarkDirectoryNrt.java ---------------------------------------------------------------------- diff --git a/src/blur-store/src/test/java/com/nearinfinity/blur/store/BenchmarkDirectoryNrt.java b/src/blur-store/src/test/java/com/nearinfinity/blur/store/BenchmarkDirectoryNrt.java index 7573e8b..ad1923d 100644 --- a/src/blur-store/src/test/java/com/nearinfinity/blur/store/BenchmarkDirectoryNrt.java +++ b/src/blur-store/src/test/java/com/nearinfinity/blur/store/BenchmarkDirectoryNrt.java @@ -1,7 +1,5 @@ package com.nearinfinity.blur.store; -import static com.nearinfinity.blur.lucene.LuceneConstant.LUCENE_VERSION; - import java.io.IOException; import java.lang.management.ManagementFactory; import java.util.Arrays; @@ -32,6 +30,8 @@ import com.nearinfinity.blur.store.blockcache.BlockDirectory; import com.nearinfinity.blur.store.blockcache.BlockDirectoryCache; import com.nearinfinity.blur.store.hdfs.HdfsDirectory; +import static com.nearinfinity.blur.lucene.LuceneConstant.LUCENE_VERSION; + public class BenchmarkDirectoryNrt { public static void main(String[] args) throws IOException, InterruptedException { @@ -131,10 +131,10 @@ public class BenchmarkDirectoryNrt { return document; } - public static int getNumberOfBanks(float heapPercentage, int numberOfBlocksPerBank, int blockSize) { + public static int getNumberOfSlabs(float heapPercentage, int numberOfBlocksPerSlab, int blockSize) { long max = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax(); long targetBytes = (long) (max * heapPercentage); - int slabSize = numberOfBlocksPerBank * blockSize; + int slabSize = numberOfBlocksPerSlab * blockSize; int slabs = (int) (targetBytes / slabSize); if (slabs == 0) { throw new RuntimeException("Minimum heap size is 512m!");
