KYLIN-1147 use pooled size for each cache(representing each realization)
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b9a19110 Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b9a19110 Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b9a19110 Branch: refs/heads/2.x-staging Commit: b9a19110b826415faddf69d76dc69c2e25e07705 Parents: 91de85c Author: honma <ho...@ebay.com> Authored: Mon Nov 16 16:08:06 2015 +0800 Committer: honma <ho...@ebay.com> Committed: Mon Nov 16 16:08:06 2015 +0800 ---------------------------------------------------------------------- .../cache/AbstractCacheFledgedQuery.java | 28 ++--- server/src/main/resources/ehcache-test.xml | 3 - server/src/main/resources/ehcache.xml | 3 - .../org/apache/kylin/rest/StorageCacheTest.java | 116 +++++++++++++++++++ 4 files changed, 131 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b9a19110/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java b/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java index a7bdae0..745ddd1 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java @@ -5,7 +5,6 @@ import net.sf.ehcache.CacheManager; import net.sf.ehcache.Element; import net.sf.ehcache.config.CacheConfiguration; import net.sf.ehcache.config.Configuration; -import net.sf.ehcache.config.MemoryUnit; import net.sf.ehcache.config.PersistenceConfiguration; import net.sf.ehcache.store.MemoryStoreEvictionPolicy; @@ -37,6 +36,14 @@ public abstract class AbstractCacheFledgedQuery implements IStorageQuery, TeeTup CACHE_MANAGER = cacheManager; } + /** + * This method is only useful non-spring injected test cases. + * When Kylin is normally ran as a spring app CACHE_MANAGER will be injected. + * and the configuration for cache lies in server/src/main/resources/ehcache.xml + * + * the cache named "StorageCache" acts like a template for each realization to + * create its own cache. + */ private static void initCacheManger() { Configuration conf = new Configuration(); conf.setMaxBytesLocalHeap("128M"); @@ -48,7 +55,7 @@ public abstract class AbstractCacheFledgedQuery implements IStorageQuery, TeeTup eternal(false).// timeToIdleSeconds(86400).// diskExpiryThreadIntervalSeconds(0).// - maxBytesLocalHeap(10, MemoryUnit.MEGABYTES).// + //maxBytesLocalHeap(10, MemoryUnit.MEGABYTES).// persistence(new PersistenceConfiguration().strategy(PersistenceConfiguration.Strategy.NONE))); CACHE_MANAGER.addCache(storageCache); @@ -84,25 +91,20 @@ public abstract class AbstractCacheFledgedQuery implements IStorageQuery, TeeTup } if (CACHE_MANAGER.getCache(storageUUID) == null) { - logger.info("Cache for {} initting...", storageUUID); + logger.info("Cache for {} initializing...", storageUUID); //Create a Cache specifying its configuration. CacheConfiguration templateConf = CACHE_MANAGER.getCache(storageCacheTemplate).getCacheConfiguration(); - PersistenceConfiguration pconf = templateConf.getPersistenceConfiguration(); - if (pconf != null) { - logger.info("PersistenceConfiguration strategy: " + pconf.getStrategy()); - } else { - logger.warn("PersistenceConfiguration is null"); - } - - Cache storageCache = new Cache(new CacheConfiguration(storageUUID, (int) templateConf.getMaxEntriesLocalHeap()).// + + Cache storageCache = new Cache(new CacheConfiguration(storageUUID, 0).// memoryStoreEvictionPolicy(templateConf.getMemoryStoreEvictionPolicy()).// eternal(templateConf.isEternal()).// timeToIdleSeconds(templateConf.getTimeToIdleSeconds()).// - maxBytesLocalHeap(templateConf.getMaxBytesLocalHeap(), MemoryUnit.BYTES).persistence(pconf)); - //TODO: deal with failed queries, and only cache too long query + //maxBytesLocalHeap(templateConf.getMaxBytesLocalHeap(), MemoryUnit.BYTES).//using pooled size + persistence(templateConf.getPersistenceConfiguration())); CACHE_MANAGER.addCache(storageCache); + } } } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b9a19110/server/src/main/resources/ehcache-test.xml ---------------------------------------------------------------------- diff --git a/server/src/main/resources/ehcache-test.xml b/server/src/main/resources/ehcache-test.xml index 8ee2246..7cae6ce 100644 --- a/server/src/main/resources/ehcache-test.xml +++ b/server/src/main/resources/ehcache-test.xml @@ -3,7 +3,6 @@ eternal="false" timeToIdleSeconds="86400" memoryStoreEvictionPolicy="LRU" - maxBytesLocalHeap="1M" > <persistence strategy="none"/> </cache> @@ -11,7 +10,6 @@ eternal="false" timeToIdleSeconds="86400" memoryStoreEvictionPolicy="LRU" - maxBytesLocalHeap="1M" > <persistence strategy="none"/> </cache> @@ -19,7 +17,6 @@ eternal="false" timeToLiveSeconds="10800" memoryStoreEvictionPolicy="LRU" - maxBytesLocalHeap="1M" > <persistence strategy="none"/> </cache> http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b9a19110/server/src/main/resources/ehcache.xml ---------------------------------------------------------------------- diff --git a/server/src/main/resources/ehcache.xml b/server/src/main/resources/ehcache.xml index 29fce05..3f39702 100644 --- a/server/src/main/resources/ehcache.xml +++ b/server/src/main/resources/ehcache.xml @@ -3,7 +3,6 @@ eternal="false" timeToIdleSeconds="86400" memoryStoreEvictionPolicy="LRU" - maxBytesLocalHeap="50M" > <persistence strategy="none"/> </cache> @@ -11,7 +10,6 @@ eternal="false" timeToIdleSeconds="86400" memoryStoreEvictionPolicy="LRU" - maxBytesLocalHeap="50M" > <persistence strategy="none"/> </cache> @@ -19,7 +17,6 @@ eternal="false" timeToLiveSeconds="10800" memoryStoreEvictionPolicy="LRU" - maxBytesLocalHeap="50M" > <persistence strategy="none"/> </cache> http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b9a19110/server/src/test/java/org/apache/kylin/rest/StorageCacheTest.java ---------------------------------------------------------------------- diff --git a/server/src/test/java/org/apache/kylin/rest/StorageCacheTest.java b/server/src/test/java/org/apache/kylin/rest/StorageCacheTest.java new file mode 100644 index 0000000..e781e00 --- /dev/null +++ b/server/src/test/java/org/apache/kylin/rest/StorageCacheTest.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.rest; + +import net.sf.ehcache.CacheManager; +import net.sf.ehcache.Element; + +import org.apache.commons.lang.NotImplementedException; +import org.apache.kylin.metadata.realization.SQLDigest; +import org.apache.kylin.metadata.tuple.ITupleIterator; +import org.apache.kylin.rest.service.ServiceTestBase; +import org.apache.kylin.storage.ICachableStorageQuery; +import org.apache.kylin.storage.StorageContext; +import org.apache.kylin.storage.cache.AbstractCacheFledgedQuery; +import org.apache.kylin.storage.cache.CacheFledgedStaticQuery; +import org.apache.kylin.storage.tuple.TupleInfo; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import com.google.common.collect.Range; + +public class StorageCacheTest extends ServiceTestBase { + + public class MockedCachableStorageQuery implements ICachableStorageQuery { + private String uuid; + + public MockedCachableStorageQuery(String uuid) { + this.uuid = uuid; + } + + @Override + public boolean isDynamic() { + return false; + } + + @Override + public Range<Long> getVolatilePeriod() { + throw new NotImplementedException(); + } + + @Override + public String getStorageUUID() { + return this.uuid; + } + + @Override + public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) { + throw new NotImplementedException(); + } + } + + @Autowired + private CacheManager cacheManager; + + @Before + public void setup() throws Exception { + AbstractCacheFledgedQuery.setCacheManager(cacheManager); + new CacheFledgedStaticQuery(new MockedCachableStorageQuery("1")); + new CacheFledgedStaticQuery(new MockedCachableStorageQuery("2")); + new CacheFledgedStaticQuery(new MockedCachableStorageQuery("3")); + new CacheFledgedStaticQuery(new MockedCachableStorageQuery("4")); + new CacheFledgedStaticQuery(new MockedCachableStorageQuery("5")); + new CacheFledgedStaticQuery(new MockedCachableStorageQuery("6")); + new CacheFledgedStaticQuery(new MockedCachableStorageQuery("7")); + new CacheFledgedStaticQuery(new MockedCachableStorageQuery("8")); + new CacheFledgedStaticQuery(new MockedCachableStorageQuery("9")); + new CacheFledgedStaticQuery(new MockedCachableStorageQuery("10")); + new CacheFledgedStaticQuery(new MockedCachableStorageQuery("11")); + } + + @Test + public void test1() { + int oneM = 1 << 20; + cacheManager.getCache("1").put(new Element("xx", new byte[oneM])); + Element xx = cacheManager.getCache("1").get("xx"); + Assert.assertEquals(oneM, ((byte[]) xx.getObjectValue()).length); + + cacheManager.getCache("2").put(new Element("yy", new byte[3 * oneM])); + Element yy = cacheManager.getCache("2").get("yy"); + Assert.assertEquals(3 * oneM, ((byte[]) yy.getObjectValue()).length); + + cacheManager.getCache("3").put(new Element("zz", new byte[10 * oneM])); + Element zz = cacheManager.getCache("3").get("zz"); + Assert.assertEquals(null, zz); + + cacheManager.getCache("4").put(new Element("aa", new byte[oneM])); + Element aa = cacheManager.getCache("4").get("aa"); + Assert.assertEquals(oneM, ((byte[]) aa.getObjectValue()).length); + + cacheManager.getCache("2").put(new Element("bb", new byte[3 * oneM])); + Element bb = cacheManager.getCache("2").get("bb"); + Assert.assertEquals(3 * oneM, ((byte[]) bb.getObjectValue()).length); + } + + @Test + public void test2() { + } +}
