Repository: incubator-carbondata Updated Branches: refs/heads/master 3966f9927 -> ff182b769
[CARBONDATA-890] For Spark 2.1 LRU cache size at driver is getting configured with the executor lru cache size. Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/99582a76 Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/99582a76 Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/99582a76 Branch: refs/heads/master Commit: 99582a762871c8a2c6cc9f5db43718d21e13de0c Parents: 3966f99 Author: mohammadshahidkhan <mohdshahidkhan1...@gmail.com> Authored: Tue Apr 11 09:59:45 2017 +0530 Committer: ravipesala <ravi.pes...@gmail.com> Committed: Wed Apr 12 14:10:34 2017 +0530 ---------------------------------------------------------------------- .../core/cache/CacheProviderTest.java | 53 ++++++++++++++++++++ .../scala/org/apache/spark/sql/CarbonEnv.scala | 1 + 2 files changed, 54 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/99582a76/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java b/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java index 1be5cb2..4c12259 100644 --- a/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java +++ b/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java @@ -17,16 +17,26 @@ package org.apache.carbondata.core.cache; +import java.io.IOException; +import java.lang.reflect.Field; + import org.apache.carbondata.core.cache.dictionary.Dictionary; import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier; import org.apache.carbondata.core.cache.dictionary.ForwardDictionaryCache; import org.apache.carbondata.core.cache.dictionary.ReverseDictionaryCache; import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.datastore.BlockIndexStore; +import org.apache.carbondata.core.datastore.SegmentTaskIndexStore; +import org.apache.carbondata.core.datastore.TableSegmentUniqueIdentifier; +import org.apache.carbondata.core.datastore.block.SegmentTaskIndexWrapper; +import org.apache.carbondata.core.datastore.block.TableBlockUniqueIdentifier; import org.apache.carbondata.core.util.CarbonProperties; +import org.apache.avro.Schema; import org.junit.Before; import org.junit.Test; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -39,6 +49,9 @@ public class CacheProviderTest { // enable lru cache by setting cache size CarbonProperties.getInstance() .addProperty(CarbonCommonConstants.CARBON_MAX_DRIVER_LRU_CACHE_SIZE, "10"); + // enable lru cache by setting cache size + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_MAX_EXECUTOR_LRU_CACHE_SIZE, "20"); } @Test public void getInstance() throws Exception { @@ -62,4 +75,44 @@ public class CacheProviderTest { assertTrue(reverseDictionaryCache instanceof ReverseDictionaryCache); assertFalse(reverseDictionaryCache instanceof ForwardDictionaryCache); } + + /** + * to test the driver and executor lru memory configuration + * + * @throws IOException + * @throws NoSuchFieldException + * @throws IllegalAccessException + */ + @Test public void driverExecutorCacheConfTest() + throws IOException, NoSuchFieldException, IllegalAccessException { + // get cache provider instance + CacheProvider cacheProvider = CacheProvider.getInstance(); + CarbonProperties.getInstance().addProperty(CarbonCommonConstants.IS_DRIVER_INSTANCE, "true"); + Cache<TableSegmentUniqueIdentifier, SegmentTaskIndexStore> driverCache = + cacheProvider.createCache(CacheType.DRIVER_BTREE, "carbonStore"); + Field carbonLRUCacheField = SegmentTaskIndexStore.class.getDeclaredField("lruCache"); + carbonLRUCacheField.setAccessible(true); + CarbonLRUCache carbonLRUCache = (CarbonLRUCache) carbonLRUCacheField.get(driverCache); + Field lruCacheMemorySizeField = CarbonLRUCache.class.getDeclaredField("lruCacheMemorySize"); + lruCacheMemorySizeField.setAccessible(true); + long lruCacheMemorySize = (long) lruCacheMemorySizeField.get(carbonLRUCache); + String driverCacheSize = CarbonProperties.getInstance() + .getProperty(CarbonCommonConstants.CARBON_MAX_DRIVER_LRU_CACHE_SIZE); + assertEquals(1024 * 1024 * Integer.parseInt(driverCacheSize), lruCacheMemorySize); + // drop cache + cacheProvider.dropAllCache(); + // validation test for the executor memory. + CarbonProperties.getInstance().addProperty(CarbonCommonConstants.IS_DRIVER_INSTANCE, "false"); + Cache<TableBlockUniqueIdentifier, BlockIndexStore> executorCache = + cacheProvider.createCache(CacheType.EXECUTOR_BTREE, "carbonStore"); + carbonLRUCacheField = BlockIndexStore.class.getSuperclass().getDeclaredField("lruCache"); + carbonLRUCacheField.setAccessible(true); + carbonLRUCache = (CarbonLRUCache) carbonLRUCacheField.get(executorCache); + lruCacheMemorySizeField = CarbonLRUCache.class.getDeclaredField("lruCacheMemorySize"); + lruCacheMemorySizeField.setAccessible(true); + lruCacheMemorySize = (long) lruCacheMemorySizeField.get(carbonLRUCache); + String executorCacheSize = CarbonProperties.getInstance() + .getProperty(CarbonCommonConstants.CARBON_MAX_EXECUTOR_LRU_CACHE_SIZE); + assertEquals(1024 * 1024 * Integer.parseInt(executorCacheSize), lruCacheMemorySize); + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/99582a76/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala index 976a1b1..d426348 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala @@ -49,6 +49,7 @@ object CarbonEnv { LOGGER.info(s"carbon env initial: $storePath") new CarbonMetastore(sparkSession.conf, storePath) } + CarbonProperties.getInstance.addProperty(CarbonCommonConstants.IS_DRIVER_INSTANCE, "true") carbonEnv = CarbonEnv(catalog) initialized = true }
