>From Hussain Towaileb <[email protected]>:
Hussain Towaileb has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18464 )
Change subject: Add compiler budget for data scan
......................................................................
Add compiler budget for data scan
Change-Id: I438e11f01b05f3ed1b8fa27468ac516f16bcdd92
---
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
M
hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetMemoryRequirementsRule.java
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
M
hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java
M
hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
M
hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
6 files changed, 56 insertions(+), 0 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/64/18464/1
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
index cfe7ce8..33e87f0 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
@@ -62,6 +62,10 @@
INTEGER_BYTE_UNIT,
StorageUtil.getIntSizeInBytes(8, KILOBYTE),
"The memory budget (in bytes) for an external scan operator
instance in a partition"),
+ COMPILER_DATASCANMEMORY(
+ INTEGER_BYTE_UNIT,
+ StorageUtil.getIntSizeInBytes(256, MEGABYTE),
+ "The memory budget (in bytes) for data scan scan operator
instance in a partition"),
COMPILER_FRAMESIZE(
INTEGER_BYTE_UNIT,
StorageUtil.getIntSizeInBytes(32, KILOBYTE),
@@ -202,6 +206,8 @@
public static final String COMPILER_EXTERNALSCANMEMORY_KEY =
Option.COMPILER_EXTERNALSCANMEMORY.ini();
+ public static final String COMPILER_DATASCANMEMORY_KEY =
Option.COMPILER_DATASCANMEMORY.ini();
+
public static final String COMPILER_BATCH_LOOKUP_KEY =
Option.COMPILER_BATCH_LOOKUP.ini();
public static final String COMPILER_CBO_KEY = Option.COMPILER_CBO.ini();
@@ -288,6 +294,10 @@
return accessor.getInt(Option.COMPILER_EXTERNALSCANMEMORY);
}
+ public int getDataScanMemorySize() {
+ return accessor.getInt(Option.COMPILER_DATASCANMEMORY);
+ }
+
public boolean isBatchLookup() {
return accessor.getBoolean(Option.COMPILER_BATCH_LOOKUP);
}
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
index c83b86e..b728390 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
@@ -80,6 +80,9 @@
CompilerProperties.COMPILER_MIN_MEMORY_ALLOCATION_KEY,
compilerProperties.getMinMemoryAllocation());
boolean arrayIndex = getBoolean(querySpecificConfig,
CompilerProperties.COMPILER_ARRAYINDEX_KEY,
compilerProperties.isArrayIndex());
+ int dataScanBufferSize = getDataScanBufferSize(
+ (String)
querySpecificConfig.get(CompilerProperties.COMPILER_DATASCANMEMORY_KEY),
+ compilerProperties.getDataScanMemorySize(), sourceLoc);
int externalScanBufferSize = getExternalScanBufferSize(
(String)
querySpecificConfig.get(CompilerProperties.COMPILER_EXTERNALSCANMEMORY_KEY),
compilerProperties.getExternalScanMemorySize(), sourceLoc);
@@ -112,6 +115,7 @@
physOptConf.setSubplanNestedPushdown(subplanNestedPushdown);
physOptConf.setMinMemoryAllocation(minMemoryAllocation);
physOptConf.setArrayIndexEnabled(arrayIndex);
+ physOptConf.setDataScanBufferSize(dataScanBufferSize);
physOptConf.setExternalScanBufferSize(externalScanBufferSize);
physOptConf.setBatchLookup(batchLookup);
physOptConf.setCBOMode(cbo);
@@ -133,6 +137,17 @@
return physOptConf;
}
+ private static int getDataScanBufferSize(String
dataScanMemorySizeParameter,
+ int compilerDataScanMemorySize,
SourceLocation sourceLoc) throws AsterixException {
+ IOptionType<Integer> intByteParser = OptionTypes.INTEGER_BYTE_UNIT;
+ try {
+ return dataScanMemorySizeParameter != null ?
intByteParser.parse(dataScanMemorySizeParameter)
+ : compilerDataScanMemorySize;
+ } catch (IllegalArgumentException e) {
+ throw AsterixException.create(ErrorCode.COMPILATION_ERROR,
sourceLoc, e.getMessage());
+ }
+ }
+
private static int getExternalScanBufferSize(String
externalScanMemorySizeParameter,
int compilerExternalScanMemorySize, SourceLocation sourceLoc)
throws AsterixException {
IOptionType<Integer> intByteParser = OptionTypes.INTEGER_BYTE_UNIT;
diff --git
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java
index 03cb83b..33610eb 100644
---
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java
+++
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java
@@ -39,6 +39,7 @@
import org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain;
import
org.apache.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
import
org.apache.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import
org.apache.hyracks.algebricks.core.algebra.properties.LocalMemoryRequirements;
import
org.apache.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
import
org.apache.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
@@ -48,6 +49,9 @@
@SuppressWarnings("rawtypes")
public class DataSourceScanPOperator extends AbstractScanPOperator {
+ // variable memory, min 1 frames
+ public static final int MIN_FRAME_LIMIT_FOR_DATA_SCAN = 1;
+
private final IDataSource<?> dataSource;
private Object implConfig;
@@ -131,6 +135,11 @@
}
@Override
+ public void createLocalMemoryRequirements(ILogicalOperator op) {
+ localMemoryRequirements =
LocalMemoryRequirements.variableMemoryBudget(MIN_FRAME_LIMIT_FOR_DATA_SCAN);
+ }
+
+ @Override
public String toString(boolean verbose) {
String ss = super.toString(verbose);
if (verbose) {
diff --git
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
index 98c4223..b282a45 100644
---
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
+++
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
@@ -44,6 +44,8 @@
public static final String QUERY_PLAN_SHAPE_DEFAULT =
QUERY_PLAN_SHAPE_ZIGZAG;
public static final int EXTERNAL_SCAN_BUFFER_SIZE =
StorageUtil.getIntSizeInBytes(8, StorageUtil.StorageUnit.KILOBYTE);
+ public static final int DATA_SCAN_BUFFER_SIZE =
+ StorageUtil.getIntSizeInBytes(256,
StorageUtil.StorageUnit.MEGABYTE);
public static final boolean BATCH_LOOKUP_DEFAULT = true;
public static final boolean COLUMN_FILTER_DEFAULT = true;
}
diff --git
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
index d167153..be02ea2 100644
---
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
+++
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
@@ -47,6 +47,7 @@
private static final String MIN_MEMORY_ALLOCATION =
"MIN_MEMORY_ALLOCATION";
private static final String ARRAY_INDEX = "ARRAY_INDEX";
private static final String EXTERNAL_SCAN_BUFFER_SIZE =
"EXTERNAL_SCAN_BUFFER_SIZE";
+ private static final String DATA_SCAN_BUFFER_SIZE =
"DATA_SCAN_BUFFER_SIZE";
private static final String BATCH_LOOKUP = "BATCH_LOOKUP";
private static final String CBO = "CBO";
private static final String CBO_TEST = "CBO_TEST";
@@ -241,6 +242,11 @@
setBoolean(ARRAY_INDEX, arrayIndex);
}
+ public int getDataScanBufferSizeInFrames() {
+ int frameSize = getFrameSize();
+ return (int) (((long) getInt(DATA_SCAN_BUFFER_SIZE,
AlgebricksConfig.DATA_SCAN_BUFFER_SIZE)) / frameSize);
+ }
+
public int getExternalScanBufferSize() {
return getInt(EXTERNAL_SCAN_BUFFER_SIZE,
AlgebricksConfig.EXTERNAL_SCAN_BUFFER_SIZE);
}
@@ -291,6 +297,10 @@
setBoolean(BATCH_LOOKUP, batchedLookup);
}
+ public void setDataScanBufferSize(int bufferSize) {
+ setInt(DATA_SCAN_BUFFER_SIZE, bufferSize);
+ }
+
public void setExternalScanBufferSize(int bufferSize) {
setInt(EXTERNAL_SCAN_BUFFER_SIZE, bufferSize);
}
diff --git
a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetMemoryRequirementsRule.java
b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetMemoryRequirementsRule.java
index 4e859e4..0ba58cd 100644
---
a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetMemoryRequirementsRule.java
+++
b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetMemoryRequirementsRule.java
@@ -318,6 +318,7 @@
@Override
public Void visitDataScanOperator(DataSourceScanOperator op, Void arg)
throws AlgebricksException {
+ setOperatorMemoryBudget(op,
physConfig.getDataScanBufferSizeInFrames());
return null;
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18464
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I438e11f01b05f3ed1b8fa27468ac516f16bcdd92
Gerrit-Change-Number: 18464
Gerrit-PatchSet: 1
Gerrit-Owner: Hussain Towaileb <[email protected]>
Gerrit-MessageType: newchange
