Repository: carbondata Updated Branches: refs/heads/master 6374d361b -> 32405f4f5
[CARBONDATA-2300] Add ENABLE_UNSAFE_IN_QUERY_EXECUTION as a configuration parameter in presto integration Add ENABLE_UNSAFE_IN_QUERY_EXECUTION as a configuration parameter in presto integration Provide this configuration parameter for users to disable the unsafe in query execution. This closes #2126 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/32405f4f Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/32405f4f Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/32405f4f Branch: refs/heads/master Commit: 32405f4f5b402836e1046047beb6d1ea54c26c15 Parents: 6374d36 Author: chenliang613 <[email protected]> Authored: Sun Apr 1 13:06:37 2018 +0800 Committer: Jacky Li <[email protected]> Committed: Mon Apr 2 23:39:32 2018 +0800 ---------------------------------------------------------------------- integration/presto/README.md | 14 +++++++++----- .../carbondata/presto/impl/CarbonTableConfig.java | 11 +++++++++++ .../carbondata/presto/impl/CarbonTableReader.java | 8 +++++++- 3 files changed, 27 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/32405f4f/integration/presto/README.md ---------------------------------------------------------------------- diff --git a/integration/presto/README.md b/integration/presto/README.md index e1d4631..8da70d7 100644 --- a/integration/presto/README.md +++ b/integration/presto/README.md @@ -18,7 +18,7 @@ Please follow the below steps to query carbondata in presto ### Config presto server -* Download presto server (0.186 is suggested and supported) : https://repo1.maven.org/maven2/com/facebook/presto/presto-server/ +* Download presto server (0.187 is suggested and supported) : https://repo1.maven.org/maven2/com/facebook/presto/presto-server/ * Finish presto configuration following https://prestodb.io/docs/current/installation/deployment.html. A configuration example: ``` @@ -30,6 +30,8 @@ Please follow the below steps to query carbondata in presto query.max-memory-per-node=1GB discovery-server.enabled=true discovery.uri=http://localhost:8086 + reorder-joins=true + jvm.config: -server @@ -60,9 +62,9 @@ Please follow the below steps to query carbondata in presto $ mvn -DskipTests -P{spark-version} -Dspark.version={spark-version-number} -Dhadoop.version={hadoop-version-number} clean package ``` Replace the spark and hadoop version with the version used in your cluster. - For example, if you are using Spark 2.1.0 and Hadoop 2.7.2, you would like to compile using: + For example, if you are using Spark 2.2.1 and Hadoop 2.7.2, you would like to compile using: ``` - mvn -DskipTests -Pspark-2.1 -Dspark.version=2.1.0 -Dhadoop.version=2.7.2 clean package + mvn -DskipTests -Pspark-2.2 -Dspark.version=2.2.1 -Dhadoop.version=2.7.2 clean package ``` Secondly: Create a folder named 'carbondata' under $PRESTO_HOME$/plugin and @@ -73,14 +75,16 @@ Please follow the below steps to query carbondata in presto ``` connector.name=carbondata carbondata-store={schema-store-path} + enable.unsafe.in.query.processing=false carbon.unsafe.working.memory.in.mb={value} ``` Replace the schema-store-path with the absolute path of the parent directory of the schema. For example, if you have a schema named 'default' stored in hdfs://namenode:9000/test/carbondata/, Then set carbondata-store=hdfs://namenode:9000/test/carbondata - carbon.unsafe.working.memory.in.mb property defines the limit for Unsafe Memory usage in Mega Bytes. Replace the value for it - as per your need, if your tables are big you can increase the unsafe memory. The default value is 512 MB. + enable.unsafe.in.query.processing property by default is true in CarbonData system, the carbon.unsafe.working.memory.in.mb + property defines the limit for Unsafe Memory usage in Mega Bytes, the default value is 512 MB. + If your tables are big you can increase the unsafe memory, or disable unsafe via setting enable.unsafe.in.query.processing=false. If you updated the jar balls or configuration files, make sure you have dispatched them to all the presto nodes and restarted the presto servers on the nodes. The updates will not take effect before restarting. http://git-wip-us.apache.org/repos/asf/carbondata/blob/32405f4f/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java index 73f9d2c..6a96221 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java @@ -31,6 +31,7 @@ public class CarbonTableConfig { private String tablePath; private String storePath; private String unsafeMemoryInMb; + private String enableUnsafeInQueryExecution; @NotNull public String getDbPath() { return dbPath; @@ -68,4 +69,14 @@ public class CarbonTableConfig { this.unsafeMemoryInMb = unsafeMemoryInMb; return this; } + + public String getEnableUnsafeInQueryExecution() { + return enableUnsafeInQueryExecution; + } + + @Config("enable.unsafe.in.query.processing") + public CarbonTableConfig setEnableUnsafeInQueryExecution(String enableUnsafeInQueryExecution) { + this.enableUnsafeInQueryExecution = enableUnsafeInQueryExecution; + return this; + } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/32405f4f/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java index 78dbbec..fa3fb9c 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java @@ -369,9 +369,15 @@ public class CarbonTableReader { Expression filters) { List<CarbonLocalInputSplit> result = new ArrayList<>(); if(config.getUnsafeMemoryInMb() != null) { - CarbonProperties.getInstance().addProperty(CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB, + CarbonProperties.getInstance().addProperty( + CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB, config.getUnsafeMemoryInMb()); } + if(config.getEnableUnsafeInQueryExecution() != null) { + CarbonProperties.getInstance().addProperty( + CarbonCommonConstants.ENABLE_UNSAFE_IN_QUERY_EXECUTION, + config.getEnableUnsafeInQueryExecution()); + } CarbonTable carbonTable = tableCacheModel.carbonTable; TableInfo tableInfo = tableCacheModel.carbonTable.getTableInfo(); Configuration config = new Configuration();
