This is an automated email from the ASF dual-hosted git repository.
zhangzc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new cb7e189 [CARBONDATA-3509] Support disable query prefetch by
configuration
cb7e189 is described below
commit cb7e18944a23f598b18370b86dd33736382a18aa
Author: ajantha-bhat <[email protected]>
AuthorDate: Fri Aug 30 11:08:09 2019 +0530
[CARBONDATA-3509] Support disable query prefetch by configuration
Support disable query prefetch by configuration:
Prefetch runs in asynch thread in query and it is always enable in query
flow. If some user wants to disable it, they can use this property to disable
and observe in logs.
This closes #3370
---
.../carbondata/core/constants/CarbonCommonConstants.java | 9 +++++++++
.../core/scan/executor/impl/AbstractQueryExecutor.java | 2 ++
.../apache/carbondata/core/scan/model/QueryModel.java | 4 +++-
.../apache/carbondata/core/util/CarbonProperties.java | 16 ++++++++++++++++
docs/configuration-parameters.md | 1 +
5 files changed, 31 insertions(+), 1 deletion(-)
diff --git
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 17b191d..67fa13f 100644
---
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1483,6 +1483,15 @@ public final class CarbonCommonConstants {
public static final String
CARBON_MAX_EXECUTOR_THREADS_FOR_BLOCK_PRUNING_DEFAULT = "4";
+ /*
+ * whether to enable prefetch for query
+ */
+ @CarbonProperty
+ public static final String CARBON_QUERY_PREFETCH_ENABLE =
+ "carbon.query.prefetch.enable";
+
+ public static final String CARBON_QUERY_PREFETCH_ENABLE_DEFAULT = "true";
+
//////////////////////////////////////////////////////////////////////////////////////////
// Datamap parameter start here
//////////////////////////////////////////////////////////////////////////////////////////
diff --git
a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
index b3d4780..6760e77 100644
---
a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
+++
b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
@@ -493,7 +493,9 @@ public abstract class AbstractQueryExecutor<E> implements
QueryExecutor<E> {
segmentProperties.getDimensionOrdinalToChunkMapping().size());
if (queryModel.isReadPageByPage()) {
blockExecutionInfo.setPrefetchBlocklet(false);
+ LOGGER.info("Query prefetch is: false, read page by page");
} else {
+ LOGGER.info("Query prefetch is: " + queryModel.isPreFetchData());
blockExecutionInfo.setPrefetchBlocklet(queryModel.isPreFetchData());
}
// In case of fg datamap it should not go to direct fill.
diff --git
a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
index 267527f..4d10492 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
@@ -34,6 +34,7 @@ import
org.apache.carbondata.core.scan.expression.UnknownExpression;
import
org.apache.carbondata.core.scan.expression.conditional.ConditionalExpression;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
import org.apache.carbondata.core.stats.QueryStatisticsRecorder;
+import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataTypeConverter;
@@ -110,7 +111,7 @@ public class QueryModel {
// whether to clear/free unsafe memory or not
private boolean freeUnsafeMemory = true;
- private boolean preFetchData = true;
+ private boolean preFetchData;
/**
* It fills the vector directly from decoded column page with out any
staging and conversions.
@@ -125,6 +126,7 @@ public class QueryModel {
tableBlockInfos = new ArrayList<TableBlockInfo>();
this.table = carbonTable;
this.queryId = String.valueOf(System.nanoTime());
+ this.preFetchData = CarbonProperties.getQueryPrefetchEnable();
}
public static QueryModel newInstance(CarbonTable carbonTable) {
diff --git
a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index c60dad8..adf4905 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -1773,4 +1773,20 @@ public final class CarbonProperties {
}
}
}
+
+ /**
+ * Validate and get query prefetch enable
+ *
+ * @return boolean prefetch value
+ */
+ public static Boolean getQueryPrefetchEnable() {
+ String prefetchEnable = CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.CARBON_QUERY_PREFETCH_ENABLE);
+ if (prefetchEnable == null) {
+ return
Boolean.parseBoolean(CarbonCommonConstants.CARBON_QUERY_PREFETCH_ENABLE_DEFAULT);
+ } else {
+ // return false only if false is set. any other case return true
+ return !prefetchEnable.equalsIgnoreCase("false");
+ }
+ }
}
diff --git a/docs/configuration-parameters.md b/docs/configuration-parameters.md
index 808d507..da226ec 100644
--- a/docs/configuration-parameters.md
+++ b/docs/configuration-parameters.md
@@ -142,6 +142,7 @@ This section provides the details of all the configurations
required for the Car
| carbon.max.driver.threads.for.block.pruning | 4 | Number of threads used for
driver pruning when the carbon files are more than 100k Maximum memory. This
configuration can used to set number of threads between 1 to 4. |
| carbon.heap.memory.pooling.threshold.bytes | 1048576 | CarbonData supports
unsafe operations of Java to avoid GC overhead for certain operations. Using
unsafe, memory can be allocated on Java Heap or off heap. This configuration
controls the allocation mechanism on Java HEAP. If the heap memory allocations
of the given size is greater or equal than this value,it should go through the
pooling mechanism. But if set this size to -1, it should not go through the
pooling mechanism. Default [...]
| carbon.push.rowfilters.for.vector | false | When enabled complete row
filters will be handled by carbon in case of vector. If it is disabled then
only page level pruning will be done by carbon and row level filtering will be
done by spark for vector. And also there are scan optimizations in carbon to
avoid multiple data copies when this parameter is set to false. There is no
change in flow for non-vector based queries. |
+| carbon.query.prefetch.enable | true | By default this property is true, so
prefetch is used in query to read next blocklet asynchronously in other thread
while processing current blocklet in main thread. This can help to reduce CPU
idle time. Setting this property false will disable this prefetch feature in
query. |
## Data Mutation Configuration
| Parameter | Default Value | Description |
