This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 43e46f3 [CARBONDATA-3959] Added new property to set the value of
executor LRU cache size to 70% of the total executor memory in IndexServer, if
executor LRU cache size is not configured
43e46f3 is described below
commit 43e46f3be275b98a3d6a37f44b212782cef22c87
Author: Karan980 <[email protected]>
AuthorDate: Mon Aug 17 18:46:53 2020 +0530
[CARBONDATA-3959] Added new property to set the value of executor
LRU cache size to 70% of the total executor memory in IndexServer,
if executor LRU cache size is not configured
Why is this PR needed?
This PR will set executor LRU cache memory to 70% of executor memory
size, if it is not configured.
What changes were proposed in this PR?
Added new property to set executor LRU cache size to 70%
This closes #3893
---
.../carbondata/core/cache/CacheProvider.java | 38 ++++++++++++++++------
.../core/constants/CarbonCommonConstants.java | 13 ++++++++
docs/index-server.md | 3 +-
.../carbondata/indexserver/IndexServer.scala | 3 --
4 files changed, 43 insertions(+), 14 deletions(-)
diff --git
a/core/src/main/java/org/apache/carbondata/core/cache/CacheProvider.java
b/core/src/main/java/org/apache/carbondata/core/cache/CacheProvider.java
index a9cf59e..e9316ac 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/CacheProvider.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/CacheProvider.java
@@ -148,19 +148,37 @@ public class CacheProvider {
carbonLRUCache = new
CarbonLRUCache(CarbonCommonConstants.CARBON_MAX_DRIVER_LRU_CACHE_SIZE,
CarbonCommonConstants.CARBON_MAX_LRU_CACHE_SIZE_DEFAULT);
} else {
- // if executor cache size is not configured then driver cache conf will
be used
String executorCacheSize = CarbonProperties.getInstance()
.getProperty(CarbonCommonConstants.CARBON_MAX_EXECUTOR_LRU_CACHE_SIZE);
- if (null != executorCacheSize) {
- carbonLRUCache =
- new
CarbonLRUCache(CarbonCommonConstants.CARBON_MAX_EXECUTOR_LRU_CACHE_SIZE,
- CarbonCommonConstants.CARBON_MAX_LRU_CACHE_SIZE_DEFAULT);
- } else {
- LOGGER.info(
- "Executor LRU cache size not configured. Initializing with driver
LRU cache size.");
- carbonLRUCache = new
CarbonLRUCache(CarbonCommonConstants.CARBON_MAX_DRIVER_LRU_CACHE_SIZE,
- CarbonCommonConstants.CARBON_MAX_LRU_CACHE_SIZE_DEFAULT);
+ if (null == executorCacheSize) {
+ String executorCachePercent = CarbonProperties.getInstance()
+
.getProperty(CarbonCommonConstants.CARBON_EXECUTOR_LRU_CACHE_PERCENT);
+ long mSizeMB = Runtime.getRuntime().maxMemory() / (1024 * 1024);
+ long executorLruCache;
+ if (null != executorCachePercent) {
+ int percentValue = Integer.parseInt(executorCachePercent);
+ if (percentValue >= 5 && percentValue <= 95) {
+ double mPercent = (double) percentValue / 100;
+ executorLruCache = (long) (mSizeMB * mPercent);
+ } else {
+ LOGGER.info("Illegal value provided to Executor LRU cache percent.
" +
+ "Initializing with default executor LRU cache percentage
size.");
+ executorLruCache = (long) (mSizeMB *
+
CarbonCommonConstants.CARBON_DEFAULT_EXECUTOR_LRU_CACHE_PERCENT);
+ }
+ } else {
+ LOGGER.info("Value of Executor LRU cache percentage is not provided.
" +
+ "Initializing with default executor LRU cache percentage
size.");
+ executorLruCache = (long) (mSizeMB *
+
CarbonCommonConstants.CARBON_DEFAULT_EXECUTOR_LRU_CACHE_PERCENT);
+ }
+ CarbonProperties.getInstance()
+
.addProperty(CarbonCommonConstants.CARBON_MAX_EXECUTOR_LRU_CACHE_SIZE,
+ String.valueOf(executorLruCache));
}
+ carbonLRUCache =
+ new
CarbonLRUCache(CarbonCommonConstants.CARBON_MAX_EXECUTOR_LRU_CACHE_SIZE,
+ CarbonCommonConstants.CARBON_MAX_LRU_CACHE_SIZE_DEFAULT);
}
}
diff --git
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index bdb3558..cde3efa 100644
---
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1281,6 +1281,19 @@ public final class CarbonCommonConstants {
"carbon.max.executor.lru.cache.size";
/**
+ * executor lru cache percent upto which lru cache will be loaded in memory.
+ * Value lies between (1 - 100)
+ */
+ @CarbonProperty
+ public static final String CARBON_EXECUTOR_LRU_CACHE_PERCENT =
+ "carbon.executor.lru.cache.percent";
+
+ /**
+ * when executor LRU cache is not configured, set it to 70% percent of
executor memory size
+ */
+ public static final double CARBON_DEFAULT_EXECUTOR_LRU_CACHE_PERCENT = 0.7d;
+
+ /**
* max lru cache size default value in MB
*/
public static final String CARBON_MAX_LRU_CACHE_SIZE_DEFAULT = "-1";
diff --git a/docs/index-server.md b/docs/index-server.md
index e4a0299..46bc5de 100644
--- a/docs/index-server.md
+++ b/docs/index-server.md
@@ -159,7 +159,8 @@ The user can enable prepriming by using
'carbon.indexserver.enable.prepriming' =
| carbon.index.server.ip | NA | Specify the IP/HOST on which the server
would be started. Better to specify the private IP. |
| carbon.index.server.port | NA | The port on which the index server has to be
started. |
|carbon.index.server.max.worker.threads| 500 | Number of RPC handlers to open
for accepting the requests from JDBC driver. Max accepted value is Integer.Max.
Refer: [Hive
configuration](https://github.com/apache/hive/blob/master/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java#L3441)
|
-|carbon.max.executor.lru.cache.size| NA | Maximum memory **(in MB)** upto
which the executor process can cache the data (Indexes and reverse dictionary
values). Only integer values greater than 0 are accepted. **NOTE:** Mandatory
for the user to set. |
+|carbon.max.executor.lru.cache.size| NA | Maximum memory **(in MB)** upto
which the executor process can cache the data (Indexes and reverse dictionary
values). Only integer values greater than 0 are accepted. |
+|carbon.executor.lru.cache.percent| NA | Percentage of executor memory **(in
MB)** upto which the executor process can cache the data (Indexes and reverse
dictionary values). Only integer values between 5 -95 are accepted. **NOTE:**
Allocate executor LRU cache size to given percentage of total executor memory
if carbon.max.executor.lru.cache.size is not configured. If the value of this
property is not set or is set to any illegal percentage value then the executor
LRU cache size will be [...]
|carbon.index.server.max.jobname.length|50|The max length of the job to show
in the index server application UI. For bigger queries this may impact
performance as the whole string would be sent from JDBCServer to IndexServer.|
|carbon.max.executor.threads.for.block.pruning|4| max executor threads used
for block pruning. |
|carbon.index.server.inmemory.serialization.threshold.inKB|300|Max in memory
serialization size after reaching threshold data will be written to file. Min
value that the user can set is 0KB and max is 102400KB. |
diff --git
a/integration/spark/src/main/scala/org/apache/carbondata/indexserver/IndexServer.scala
b/integration/spark/src/main/scala/org/apache/carbondata/indexserver/IndexServer.scala
index 0d32007..36c2ad4 100644
---
a/integration/spark/src/main/scala/org/apache/carbondata/indexserver/IndexServer.scala
+++
b/integration/spark/src/main/scala/org/apache/carbondata/indexserver/IndexServer.scala
@@ -247,9 +247,6 @@ object IndexServer extends ServerInterface {
def main(args: Array[String]): Unit = {
if (serverIp.isEmpty) {
throw new RuntimeException(s"Please set the server IP to use Index Cache
Server")
- } else if (!isExecutorLRUConfigured) {
- throw new RuntimeException(s"Executor LRU cache size is not set. Please
set using " +
- s"${
CarbonCommonConstants.CARBON_MAX_EXECUTOR_LRU_CACHE_SIZE }")
} else {
createCarbonSession()
LOGGER.info("Starting Index Cache Server")
