This is an automated email from the ASF dual-hosted git repository.
wyk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 8d9966a4a5 [ASTERIXDB-3324][COMP][RT] Stabilize columnar filters and
projector
8d9966a4a5 is described below
commit 8d9966a4a5da00d91b081c58192416394247ecbb
Author: Wail Alkowaileet <[email protected]>
AuthorDate: Wed Dec 6 07:54:32 2023 -0800
[ASTERIXDB-3324][COMP][RT] Stabilize columnar filters and projector
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
This patch includes multiple fixes described in ASTERIXDB-3324
Change-Id: Id8d91a207abb619b8a6a996b4446ca6e06eaaa71
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17993
Integration-Tests: Jenkins <[email protected]>
Tested-by: Jenkins <[email protected]>
Reviewed-by: Wail Alkowaileet <[email protected]>
Reviewed-by: Ali Alsuliman <[email protected]>
Contrib: Michael Blow <[email protected]>
---
.../optimizer/rules/pushdown/PushdownContext.java | 2 +-
.../processor/AbstractFilterPushdownProcessor.java | 18 +++++++++-------
.../processor/ColumnFilterPushdownProcessor.java | 9 ++++++--
.../ColumnRangeFilterPushdownProcessor.java | 5 +++--
.../ColumnValueAccessPushdownProcessor.java | 5 ++++-
.../ExternalDatasetFilterPushdownProcessor.java | 5 +++--
.../pushdown/schema/ExpectedSchemaBuilder.java | 2 +-
.../pushdown/schema/RootExpectedSchemaNode.java | 9 ++++----
.../ExpressionValueAccessPushdownVisitor.java | 19 +++--------------
.../pushdown/visitor/PushdownOperatorVisitor.java | 16 ++++++++++++++-
.../translator/LangExpressionToPlanTranslator.java | 2 ++
.../ParquetPlainFixedLengthValuesReader.java | 6 +++---
.../column/filter/FilterAccessorProvider.java | 8 ++++++--
.../filter/NoOpColumnFilterEvaluatorFactory.java | 5 ++---
...tory.java => ColumnFilterEvaluatorContext.java} | 21 ++++++++++++-------
.../IColumnIterableFilterEvaluatorFactory.java | 5 +----
.../ColumnFilterValueAccessorEvaluatorFactory.java | 13 ++++--------
.../ColumnIterableFilterEvaluatorFactory.java | 14 +++++++++----
... => MissingColumnRangeFilterValueAccessor.java} | 6 +++---
.../NoOpColumnRangeFilterValueAccessor.java | 2 +-
.../AbstractColumnFilterComparatorFactory.java | 10 ++++++++-
.../schema/visitor/PathExtractorVisitor.java | 6 +++---
.../column/operation/query/ColumnAssembler.java | 4 ++++
.../operation/query/QueryColumnMetadata.java | 20 ++++++++----------
.../query/QueryColumnWithMetaMetadata.java | 21 ++++++++-----------
.../reader/value/key/FloatKeyValueReader.java | 2 +-
.../asterix/metadata/utils/PushdownUtil.java | 24 ++++++++++++++++++++++
.../impls/btree/AbstractColumnBTreeLeafFrame.java | 3 ++-
28 files changed, 158 insertions(+), 104 deletions(-)
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/PushdownContext.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/PushdownContext.java
index 7d1e068245..51d25a4cc2 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/PushdownContext.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/PushdownContext.java
@@ -244,7 +244,7 @@ public class PushdownContext {
private static Set<LogicalOperatorTag> getScopeOperators() {
return EnumSet.of(LogicalOperatorTag.INNERJOIN,
LogicalOperatorTag.LEFTOUTERJOIN, LogicalOperatorTag.GROUP,
- LogicalOperatorTag.AGGREGATE, LogicalOperatorTag.WINDOW);
+ LogicalOperatorTag.AGGREGATE, LogicalOperatorTag.WINDOW,
LogicalOperatorTag.UNIONALL);
}
}
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/processor/AbstractFilterPushdownProcessor.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/processor/AbstractFilterPushdownProcessor.java
index 9a88036db6..02870c5508 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/processor/AbstractFilterPushdownProcessor.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/processor/AbstractFilterPushdownProcessor.java
@@ -84,9 +84,11 @@ abstract class AbstractFilterPushdownProcessor extends
AbstractPushdownProcessor
/**
* Prepare to pushdown a SELECT expression in the use-descriptor
*
- * @param useDescriptor contains the SELECT operator and its expression
+ * @param useDescriptor contains the operator and its expression
+ * @param scanDescriptor contains the scan definition where to push the
filter expression
*/
- protected abstract void preparePushdown(UseDescriptor useDescriptor)
throws AlgebricksException;
+ protected abstract void preparePushdown(UseDescriptor useDescriptor,
ScanDefineDescriptor scanDescriptor)
+ throws AlgebricksException;
/**
* Is an expression pushable
@@ -159,23 +161,25 @@ abstract class AbstractFilterPushdownProcessor extends
AbstractPushdownProcessor
private boolean inlineAndPushdownFilter(UseDescriptor useDescriptor,
ScanDefineDescriptor scanDefineDescriptor)
throws AlgebricksException {
- ILogicalOperator selectOp = useDescriptor.getOperator();
- if (visitedOperators.contains(selectOp)) {
+ ILogicalOperator op = useDescriptor.getOperator();
+ if (visitedOperators.contains(op)) {
// Skip and follow through to find any other selects that can be
pushed down
return false;
}
boolean changed = false;
- // Get a clone of the SELECT expression and inline it
+
+ // Get a clone of the operator's expression and inline it
ILogicalExpression inlinedExpr =
pushdownContext.cloneAndInlineExpression(useDescriptor, context);
+
// Prepare for pushdown
- preparePushdown(useDescriptor);
+ preparePushdown(useDescriptor, scanDefineDescriptor);
if (pushdownFilterExpression(inlinedExpr)) {
putFilterInformation(scanDefineDescriptor, inlinedExpr);
changed = true;
}
// Do not push down a select twice.
- visitedOperators.add(selectOp);
+ visitedOperators.add(op);
return changed;
}
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/processor/ColumnFilterPushdownProcessor.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/processor/ColumnFilterPushdownProcessor.java
index 417d6cafe5..d4db751ae6 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/processor/ColumnFilterPushdownProcessor.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/processor/ColumnFilterPushdownProcessor.java
@@ -30,6 +30,7 @@ import java.util.Map;
import org.apache.asterix.common.config.DatasetConfig;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.utils.DatasetUtil;
+import org.apache.asterix.metadata.utils.PushdownUtil;
import org.apache.asterix.om.functions.BuiltinFunctions;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.optimizer.rules.pushdown.PushdownContext;
@@ -45,6 +46,7 @@ import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import
org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
@@ -92,8 +94,11 @@ public class ColumnFilterPushdownProcessor extends
AbstractFilterPushdownProcess
}
@Override
- protected void preparePushdown(UseDescriptor useDescriptor) throws
AlgebricksException {
-
exprToNodeVisitor.setTypeEnv(useDescriptor.getOperator().computeOutputTypeEnvironment(context));
+ protected void preparePushdown(UseDescriptor useDescriptor,
ScanDefineDescriptor scanDescriptor)
+ throws AlgebricksException {
+ ILogicalOperator useOp = useDescriptor.getOperator();
+ ILogicalOperator scanOp = scanDescriptor.getOperator();
+ exprToNodeVisitor.setTypeEnv(PushdownUtil.getTypeEnv(useOp, scanOp,
context));
paths.clear();
}
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/processor/ColumnRangeFilterPushdownProcessor.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/processor/ColumnRangeFilterPushdownProcessor.java
index a9dcfc126c..b56c550656 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/processor/ColumnRangeFilterPushdownProcessor.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/processor/ColumnRangeFilterPushdownProcessor.java
@@ -71,8 +71,9 @@ public class ColumnRangeFilterPushdownProcessor extends
ColumnFilterPushdownProc
}
@Override
- protected void preparePushdown(UseDescriptor useDescriptor) throws
AlgebricksException {
- super.preparePushdown(useDescriptor);
+ protected void preparePushdown(UseDescriptor useDescriptor,
ScanDefineDescriptor scanDescriptor)
+ throws AlgebricksException {
+ super.preparePushdown(useDescriptor, scanDescriptor);
sourceInformationMap.clear();
}
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/processor/ColumnValueAccessPushdownProcessor.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/processor/ColumnValueAccessPushdownProcessor.java
index 9378cff6e8..7db3dd24f3 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/processor/ColumnValueAccessPushdownProcessor.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/processor/ColumnValueAccessPushdownProcessor.java
@@ -21,6 +21,7 @@ package org.apache.asterix.optimizer.rules.pushdown.processor;
import java.util.List;
import org.apache.asterix.metadata.utils.DatasetUtil;
+import org.apache.asterix.metadata.utils.PushdownUtil;
import org.apache.asterix.optimizer.rules.pushdown.PushdownContext;
import org.apache.asterix.optimizer.rules.pushdown.descriptor.DefineDescriptor;
import
org.apache.asterix.optimizer.rules.pushdown.descriptor.ScanDefineDescriptor;
@@ -29,6 +30,7 @@ import
org.apache.asterix.optimizer.rules.pushdown.schema.ExpectedSchemaBuilder;
import
org.apache.asterix.optimizer.rules.pushdown.schema.RootExpectedSchemaNode;
import
org.apache.asterix.optimizer.rules.pushdown.visitor.ExpressionValueAccessPushdownVisitor;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
import
org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
@@ -81,7 +83,8 @@ public class ColumnValueAccessPushdownProcessor extends
AbstractPushdownProcesso
List<UseDescriptor> useDescriptors =
pushdownContext.getUseDescriptors(defineDescriptor);
for (UseDescriptor useDescriptor : useDescriptors) {
LogicalVariable producedVariable =
useDescriptor.getProducedVariable();
- IVariableTypeEnvironment typeEnv =
useDescriptor.getOperator().computeOutputTypeEnvironment(context);
+ ILogicalOperator op = useDescriptor.getOperator();
+ IVariableTypeEnvironment typeEnv = PushdownUtil.getTypeEnv(op,
context);
expressionVisitor.transform(useDescriptor.getExpression(),
producedVariable, typeEnv);
}
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/processor/ExternalDatasetFilterPushdownProcessor.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/processor/ExternalDatasetFilterPushdownProcessor.java
index bb5c8531e4..96b252a954 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/processor/ExternalDatasetFilterPushdownProcessor.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/processor/ExternalDatasetFilterPushdownProcessor.java
@@ -61,8 +61,9 @@ public class ExternalDatasetFilterPushdownProcessor extends
ColumnFilterPushdown
}
@Override
- protected void preparePushdown(UseDescriptor useDescriptor) throws
AlgebricksException {
- super.preparePushdown(useDescriptor);
+ protected void preparePushdown(UseDescriptor useDescriptor,
ScanDefineDescriptor scanDescriptor)
+ throws AlgebricksException {
+ super.preparePushdown(useDescriptor, scanDescriptor);
}
@Override
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/ExpectedSchemaBuilder.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/ExpectedSchemaBuilder.java
index 71be18e5fc..6b215084f0 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/ExpectedSchemaBuilder.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/ExpectedSchemaBuilder.java
@@ -80,7 +80,7 @@ public class ExpectedSchemaBuilder {
AbstractComplexExpectedSchemaNode parent = node.getParent();
if (parent == null) {
//It is a root node. Request the entire record
- varToNode.put(variable,
RootExpectedSchemaNode.ALL_FIELDS_ROOT_NODE);
+ varToNode.put(variable,
RootExpectedSchemaNode.ALL_FIELDS_ROOT_IRREPLACEABLE_NODE);
} else {
// If it is a nested node, replace it to a LEAF node
AnyExpectedSchemaNode leafNode = (AnyExpectedSchemaNode)
node.replaceIfNeeded(ExpectedSchemaNodeType.ANY,
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/RootExpectedSchemaNode.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/RootExpectedSchemaNode.java
index 1a978f36b4..189505b93a 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/RootExpectedSchemaNode.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/RootExpectedSchemaNode.java
@@ -27,7 +27,10 @@ public class RootExpectedSchemaNode extends
ObjectExpectedSchemaNode {
private static final int ALL_FIELDS_ROOT = 1;
//Root with custom fields
private static final int CLIPPED_ROOT = 2;
+ private static final int ALL_FIELDS_ROOT_IRREPLACEABLE = 3;
public static final RootExpectedSchemaNode ALL_FIELDS_ROOT_NODE = new
RootExpectedSchemaNode(ALL_FIELDS_ROOT);
+ public static final RootExpectedSchemaNode
ALL_FIELDS_ROOT_IRREPLACEABLE_NODE =
+ new RootExpectedSchemaNode(ALL_FIELDS_ROOT_IRREPLACEABLE);
public static final RootExpectedSchemaNode EMPTY_ROOT_NODE = new
RootExpectedSchemaNode(EMPTY_ROOT);
private final int rootType;
@@ -41,10 +44,6 @@ public class RootExpectedSchemaNode extends
ObjectExpectedSchemaNode {
this.rootType = rootType;
}
- public int getRootType() {
- return rootType;
- }
-
@Override
public AbstractComplexExpectedSchemaNode
replaceIfNeeded(ExpectedSchemaNodeType expectedNodeType,
SourceLocation sourceLocation, String functionName) {
@@ -65,6 +64,6 @@ public class RootExpectedSchemaNode extends
ObjectExpectedSchemaNode {
}
public boolean isAllFields() {
- return rootType == ALL_FIELDS_ROOT;
+ return rootType == ALL_FIELDS_ROOT || rootType ==
ALL_FIELDS_ROOT_IRREPLACEABLE;
}
}
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/visitor/ExpressionValueAccessPushdownVisitor.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/visitor/ExpressionValueAccessPushdownVisitor.java
index 45dfcfffad..a56f834991 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/visitor/ExpressionValueAccessPushdownVisitor.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/visitor/ExpressionValueAccessPushdownVisitor.java
@@ -41,21 +41,8 @@ public class ExpressionValueAccessPushdownVisitor {
this.builder = builder;
}
- public boolean transform(ILogicalExpression expression, LogicalVariable
producedVariable,
- IVariableTypeEnvironment typeEnv) throws AlgebricksException {
- return pushValueAccessExpression(expression, producedVariable,
typeEnv);
- }
-
- private boolean pushValueAccessExpression(Mutable<ILogicalExpression>
exprRef, LogicalVariable producedVar,
- IVariableTypeEnvironment typeEnv) throws AlgebricksException {
- return pushValueAccessExpression(exprRef.getValue(), producedVar,
typeEnv);
- }
-
- /**
- * Pushdown field access expressions and array access expressions down
- */
- private boolean pushValueAccessExpression(ILogicalExpression expr,
LogicalVariable producedVar,
- IVariableTypeEnvironment typeEnv) throws AlgebricksException {
+ public boolean transform(ILogicalExpression expr, LogicalVariable
producedVar, IVariableTypeEnvironment typeEnv)
+ throws AlgebricksException {
if (skipPushdown(expr)) {
return false;
}
@@ -124,7 +111,7 @@ public class ExpressionValueAccessPushdownVisitor {
* We need to set the produced variable as null here as the
produced variable will not correspond to the
* nested expression.
*/
- changed |= pushValueAccessExpression(exprRef, producedVar,
typeEnv);
+ changed |= transform(exprRef.getValue(), null, typeEnv);
}
return changed;
}
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/visitor/PushdownOperatorVisitor.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/visitor/PushdownOperatorVisitor.java
index b00f8a1e97..28309aa062 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/visitor/PushdownOperatorVisitor.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/visitor/PushdownOperatorVisitor.java
@@ -43,6 +43,7 @@ import
org.apache.asterix.optimizer.rules.pushdown.descriptor.ScanDefineDescript
import
org.apache.asterix.optimizer.rules.pushdown.schema.RootExpectedSchemaNode;
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan;
@@ -51,6 +52,7 @@ import
org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
import
org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import
org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractScanOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestMapOperator;
@@ -127,7 +129,18 @@ public class PushdownOperatorVisitor implements
ILogicalOperatorVisitor<Void, Vo
// Enter scope for (new stage) for operators like GROUP and JOIN
pushdownContext.enterScope(op);
defUseComputer.init(op, producedVariables);
+
op.acceptExpressionTransform(defUseComputer);
+ if (op.getOperatorTag() == LogicalOperatorTag.UNIONALL) {
+ // UnionAll is a special case
+ UnionAllOperator unionOp = (UnionAllOperator) op;
+ for (Triple<LogicalVariable, LogicalVariable, LogicalVariable>
vars : unionOp.getVariableMappings()) {
+ VariableReferenceExpression left = new
VariableReferenceExpression(vars.first);
+ pushdownContext.use(op, left, -1, null);
+ VariableReferenceExpression right = new
VariableReferenceExpression(vars.second);
+ pushdownContext.use(op, right, -1, null);
+ }
+ }
}
/*
@@ -212,7 +225,8 @@ public class PushdownOperatorVisitor implements
ILogicalOperatorVisitor<Void, Vo
* 2- return the actual DatasetDataSource
*/
private DatasetDataSource getDatasetDataSourceIfApplicable(DataSource
dataSource) throws AlgebricksException {
- if (dataSource == null || dataSource.getDatasourceType() ==
DataSource.Type.SAMPLE) {
+ if (dataSource == null || dataSource.getDatasourceType() ==
DataSource.Type.SAMPLE
+ || !(dataSource instanceof DatasetDataSource)) {
return null;
}
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
index 68cb2850a0..08561c0dc8 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
@@ -2198,6 +2198,8 @@ abstract class LangExpressionToPlanTranslator
NestedTupleSourceOperator ntsOp = new NestedTupleSourceOperator(new
MutableObject<>(subplanOp));
ntsOp.setSourceLocation(sourceLoc);
SelectOperator select = new SelectOperator(selectExpr);
+ // Disable pushdowns
+
select.getAnnotations().put(OperatorAnnotations.DISALLOW_FILTER_PUSHDOWN_TO_SCAN,
Boolean.TRUE);
// The select operator cannot be moved up and down, otherwise it will
cause
// typing issues (ASTERIXDB-1203).
OperatorPropertiesUtil.markMovable(select, false);
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/decoder/ParquetPlainFixedLengthValuesReader.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/decoder/ParquetPlainFixedLengthValuesReader.java
index 417043e2d8..28e0033e7b 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/decoder/ParquetPlainFixedLengthValuesReader.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/decoder/ParquetPlainFixedLengthValuesReader.java
@@ -54,7 +54,7 @@ public class ParquetPlainFixedLengthValuesReader extends
AbstractParquetValuesRe
try {
in.skipBytes(valueLength);
} catch (IOException e) {
- throw new ParquetDecodingException("could not skip double", e);
+ throw new ParquetDecodingException("could not skip " + valueLength
+ " bytes", e);
}
}
@@ -63,7 +63,7 @@ public class ParquetPlainFixedLengthValuesReader extends
AbstractParquetValuesRe
try {
return in.readLong();
} catch (IOException e) {
- throw new ParquetDecodingException("could not read double", e);
+ throw new ParquetDecodingException("could not read long", e);
}
}
@@ -72,7 +72,7 @@ public class ParquetPlainFixedLengthValuesReader extends
AbstractParquetValuesRe
try {
return in.readFloat();
} catch (IOException e) {
- throw new ParquetDecodingException("could not read double", e);
+ throw new ParquetDecodingException("could not read float", e);
}
}
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/FilterAccessorProvider.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/FilterAccessorProvider.java
index 12c0078594..af64325f60 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/FilterAccessorProvider.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/FilterAccessorProvider.java
@@ -32,6 +32,7 @@ import
org.apache.asterix.column.filter.iterable.accessor.MissingEvaluator;
import
org.apache.asterix.column.filter.iterable.accessor.UnionColumnFilterValueAccessorEvaluator;
import org.apache.asterix.column.filter.range.IColumnRangeFilterValueAccessor;
import
org.apache.asterix.column.filter.range.accessor.ColumnRangeFilterValueAccessor;
+import
org.apache.asterix.column.filter.range.accessor.MissingColumnRangeFilterValueAccessor;
import
org.apache.asterix.column.filter.range.accessor.NoOpColumnRangeFilterValueAccessor;
import org.apache.asterix.column.metadata.schema.AbstractSchemaNode;
import org.apache.asterix.column.metadata.schema.ObjectSchemaNode;
@@ -48,7 +49,6 @@ import
org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
import org.apache.hyracks.api.exceptions.HyracksDataException;
public class FilterAccessorProvider {
- public static final String FILTER_ACCESSOR_PROVIDER_KEY =
"filter-accessor-provider";
private final ObjectSchemaNode root;
private final ObjectSchemaNode metaRoot;
private final SchemaClipperVisitor clipperVisitor;
@@ -88,9 +88,13 @@ public class FilterAccessorProvider {
cachedNodes.put(path, node);
}
+ if (node.isPrimaryKey()) {
+ return NoOpColumnRangeFilterValueAccessor.INSTANCE;
+ }
+
ATypeTag typeTag = node.getTypeTag();
if (typeTag == ATypeTag.MISSING) {
- return NoOpColumnRangeFilterValueAccessor.INSTANCE;
+ return MissingColumnRangeFilterValueAccessor.INSTANCE;
}
IColumnRangeFilterValueAccessor accessor =
new ColumnRangeFilterValueAccessor(node.getColumnIndex(),
typeTag, min);
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/NoOpColumnFilterEvaluatorFactory.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/NoOpColumnFilterEvaluatorFactory.java
index 4c649a2909..5891311da7 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/NoOpColumnFilterEvaluatorFactory.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/NoOpColumnFilterEvaluatorFactory.java
@@ -18,10 +18,10 @@
*/
package org.apache.asterix.column.filter;
+import org.apache.asterix.column.filter.iterable.ColumnFilterEvaluatorContext;
import
org.apache.asterix.column.filter.iterable.IColumnIterableFilterEvaluator;
import
org.apache.asterix.column.filter.iterable.IColumnIterableFilterEvaluatorFactory;
import
org.apache.asterix.column.filter.range.IColumnRangeFilterEvaluatorFactory;
-import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
/**
@@ -41,8 +41,7 @@ public class NoOpColumnFilterEvaluatorFactory
}
@Override
- public IColumnIterableFilterEvaluator create(FilterAccessorProvider
filterAccessorProvider,
- IEvaluatorContext context) throws HyracksDataException {
+ public IColumnIterableFilterEvaluator create(ColumnFilterEvaluatorContext
context) throws HyracksDataException {
return TrueColumnFilterEvaluator.INSTANCE;
}
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/IColumnIterableFilterEvaluatorFactory.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/ColumnFilterEvaluatorContext.java
similarity index 60%
copy from
asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/IColumnIterableFilterEvaluatorFactory.java
copy to
asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/ColumnFilterEvaluatorContext.java
index 99df077e2b..b16eb68553 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/IColumnIterableFilterEvaluatorFactory.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/ColumnFilterEvaluatorContext.java
@@ -18,13 +18,20 @@
*/
package org.apache.asterix.column.filter.iterable;
-import java.io.Serializable;
-
import org.apache.asterix.column.filter.FilterAccessorProvider;
-import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.algebricks.runtime.evaluators.EvaluatorContext;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+
+public class ColumnFilterEvaluatorContext extends EvaluatorContext {
+ private final FilterAccessorProvider filterAccessorProvider;
+
+ public ColumnFilterEvaluatorContext(IHyracksTaskContext taskContext,
+ FilterAccessorProvider filterAccessorProvider) {
+ super(taskContext);
+ this.filterAccessorProvider = filterAccessorProvider;
+ }
-public interface IColumnIterableFilterEvaluatorFactory extends Serializable {
- IColumnIterableFilterEvaluator create(FilterAccessorProvider
filterAccessorProvider, IEvaluatorContext context)
- throws HyracksDataException;
+ public FilterAccessorProvider getFilterAccessorProvider() {
+ return filterAccessorProvider;
+ }
}
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/IColumnIterableFilterEvaluatorFactory.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/IColumnIterableFilterEvaluatorFactory.java
index 99df077e2b..969062214b 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/IColumnIterableFilterEvaluatorFactory.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/IColumnIterableFilterEvaluatorFactory.java
@@ -20,11 +20,8 @@ package org.apache.asterix.column.filter.iterable;
import java.io.Serializable;
-import org.apache.asterix.column.filter.FilterAccessorProvider;
-import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
public interface IColumnIterableFilterEvaluatorFactory extends Serializable {
- IColumnIterableFilterEvaluator create(FilterAccessorProvider
filterAccessorProvider, IEvaluatorContext context)
- throws HyracksDataException;
+ IColumnIterableFilterEvaluator create(ColumnFilterEvaluatorContext
context) throws HyracksDataException;
}
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/accessor/ColumnFilterValueAccessorEvaluatorFactory.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/accessor/ColumnFilterValueAccessorEvaluatorFactory.java
index ce26d6ba75..28623d78b4 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/accessor/ColumnFilterValueAccessorEvaluatorFactory.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/accessor/ColumnFilterValueAccessorEvaluatorFactory.java
@@ -19,13 +19,12 @@
package org.apache.asterix.column.filter.iterable.accessor;
import org.apache.asterix.column.filter.FilterAccessorProvider;
+import org.apache.asterix.column.filter.iterable.ColumnFilterEvaluatorContext;
import org.apache.asterix.om.types.ARecordType;
import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.dataflow.common.utils.TaskUtil;
public class ColumnFilterValueAccessorEvaluatorFactory implements
IScalarEvaluatorFactory {
private static final long serialVersionUID = -7871899093673316190L;
@@ -37,12 +36,8 @@ public class ColumnFilterValueAccessorEvaluatorFactory
implements IScalarEvaluat
@Override
public IScalarEvaluator createScalarEvaluator(IEvaluatorContext ctx)
throws HyracksDataException {
- IHyracksTaskContext taskContext = ctx.getTaskContext();
- FilterAccessorProvider provider =
-
TaskUtil.get(FilterAccessorProvider.FILTER_ACCESSOR_PROVIDER_KEY, taskContext);
- if (provider == null) {
- throw new IllegalStateException("FILTER_ACCESSOR_PROVIDER_KEY is
not set");
- }
- return provider.createColumnAccessEvaluator(path);
+ ColumnFilterEvaluatorContext columnEvalCtx =
(ColumnFilterEvaluatorContext) ctx;
+ FilterAccessorProvider filterAccessorProvider =
columnEvalCtx.getFilterAccessorProvider();
+ return filterAccessorProvider.createColumnAccessEvaluator(path);
}
}
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/evaluator/ColumnIterableFilterEvaluatorFactory.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/evaluator/ColumnIterableFilterEvaluatorFactory.java
index 742511b9f0..44441ddb86 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/evaluator/ColumnIterableFilterEvaluatorFactory.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/evaluator/ColumnIterableFilterEvaluatorFactory.java
@@ -21,10 +21,10 @@ package org.apache.asterix.column.filter.iterable.evaluator;
import java.util.List;
import org.apache.asterix.column.filter.FilterAccessorProvider;
+import org.apache.asterix.column.filter.iterable.ColumnFilterEvaluatorContext;
import
org.apache.asterix.column.filter.iterable.IColumnIterableFilterEvaluator;
import
org.apache.asterix.column.filter.iterable.IColumnIterableFilterEvaluatorFactory;
import org.apache.asterix.column.values.IColumnValuesReader;
-import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -38,10 +38,16 @@ public class ColumnIterableFilterEvaluatorFactory
implements IColumnIterableFilt
}
@Override
- public IColumnIterableFilterEvaluator create(FilterAccessorProvider
filterAccessorProvider,
- IEvaluatorContext context) throws HyracksDataException {
- List<IColumnValuesReader> readers =
filterAccessorProvider.getFilterColumnReaders();
+ public IColumnIterableFilterEvaluator create(ColumnFilterEvaluatorContext
context) throws HyracksDataException {
IScalarEvaluator evaluator =
evaluatorFactory.createScalarEvaluator(context);
+ FilterAccessorProvider filterAccessorProvider =
context.getFilterAccessorProvider();
+ // Readers are populated by evaluatorFactory.createScalarEvaluator()
+ List<IColumnValuesReader> readers =
filterAccessorProvider.getFilterColumnReaders();
+
+ if (readers.isEmpty()) {
+ throw new NullPointerException("Readers are empty");
+ }
+
if (readers.stream().anyMatch(IColumnValuesReader::isRepeated)) {
return new ColumnarRepeatedIterableFilterEvaluator(evaluator,
readers);
}
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/range/accessor/NoOpColumnRangeFilterValueAccessor.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/range/accessor/MissingColumnRangeFilterValueAccessor.java
similarity index 86%
copy from
asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/range/accessor/NoOpColumnRangeFilterValueAccessor.java
copy to
asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/range/accessor/MissingColumnRangeFilterValueAccessor.java
index 5d538c7031..5a0c7ffada 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/range/accessor/NoOpColumnRangeFilterValueAccessor.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/range/accessor/MissingColumnRangeFilterValueAccessor.java
@@ -21,10 +21,10 @@ package org.apache.asterix.column.filter.range.accessor;
import org.apache.asterix.column.filter.range.IColumnRangeFilterValueAccessor;
import org.apache.asterix.om.types.ATypeTag;
-public class NoOpColumnRangeFilterValueAccessor implements
IColumnRangeFilterValueAccessor {
- public static final IColumnRangeFilterValueAccessor INSTANCE = new
NoOpColumnRangeFilterValueAccessor();
+public class MissingColumnRangeFilterValueAccessor implements
IColumnRangeFilterValueAccessor {
+ public static final IColumnRangeFilterValueAccessor INSTANCE = new
MissingColumnRangeFilterValueAccessor();
- private NoOpColumnRangeFilterValueAccessor() {
+ private MissingColumnRangeFilterValueAccessor() {
}
@Override
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/range/accessor/NoOpColumnRangeFilterValueAccessor.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/range/accessor/NoOpColumnRangeFilterValueAccessor.java
index 5d538c7031..2718c2d918 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/range/accessor/NoOpColumnRangeFilterValueAccessor.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/range/accessor/NoOpColumnRangeFilterValueAccessor.java
@@ -34,6 +34,6 @@ public class NoOpColumnRangeFilterValueAccessor implements
IColumnRangeFilterVal
@Override
public ATypeTag getTypeTag() {
- return ATypeTag.MISSING;
+ return ATypeTag.ANY;
}
}
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/range/compartor/AbstractColumnFilterComparatorFactory.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/range/compartor/AbstractColumnFilterComparatorFactory.java
index 0374a4d433..fa9abdae1c 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/range/compartor/AbstractColumnFilterComparatorFactory.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/range/compartor/AbstractColumnFilterComparatorFactory.java
@@ -25,6 +25,7 @@ import
org.apache.asterix.column.filter.TrueColumnFilterEvaluator;
import
org.apache.asterix.column.filter.range.IColumnRangeFilterEvaluatorFactory;
import org.apache.asterix.column.filter.range.IColumnRangeFilterValueAccessor;
import
org.apache.asterix.column.filter.range.IColumnRangeFilterValueAccessorFactory;
+import
org.apache.asterix.column.filter.range.accessor.NoOpColumnRangeFilterValueAccessor;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -48,7 +49,8 @@ abstract class AbstractColumnFilterComparatorFactory
implements IColumnRangeFilt
ATypeTag leftTypeTag = leftAccessor.getTypeTag();
ATypeTag rightTypeTag = rightAccessor.getTypeTag();
- if (leftTypeTag != rightTypeTag &&
ATypeHierarchy.isCompatible(leftTypeTag, rightTypeTag)) {
+ if (isNoOp(leftAccessor, rightAccessor)
+ || leftTypeTag != rightTypeTag &&
ATypeHierarchy.isCompatible(leftTypeTag, rightTypeTag)) {
// Cannot compare comparable values with different types. Bail out.
return TrueColumnFilterEvaluator.INSTANCE;
} else if (cannotCompare(leftTypeTag, rightTypeTag)) {
@@ -57,6 +59,12 @@ abstract class AbstractColumnFilterComparatorFactory
implements IColumnRangeFilt
return createComparator(leftAccessor, rightAccessor);
}
+ private boolean isNoOp(IColumnRangeFilterValueAccessor leftAccessor,
+ IColumnRangeFilterValueAccessor rightAccessor) {
+ return leftAccessor == NoOpColumnRangeFilterValueAccessor.INSTANCE
+ || rightAccessor ==
NoOpColumnRangeFilterValueAccessor.INSTANCE;
+ }
+
private boolean cannotCompare(ATypeTag leftTypeTag, ATypeTag rightTypeTag)
{
return rightTypeTag == ATypeTag.MISSING || leftTypeTag != rightTypeTag;
}
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/visitor/PathExtractorVisitor.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/visitor/PathExtractorVisitor.java
index efff918769..1b63361b94 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/visitor/PathExtractorVisitor.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/visitor/PathExtractorVisitor.java
@@ -113,7 +113,6 @@ public class PathExtractorVisitor implements
ISchemaNodeVisitor<AbstractSchemaNo
@Override
public AbstractSchemaNode visit(PrimitiveSchemaNode primitiveNode, Void
arg) throws HyracksDataException {
- //Missing column index is -1
return primitiveNode;
}
@@ -126,8 +125,9 @@ public class PathExtractorVisitor implements
ISchemaNodeVisitor<AbstractSchemaNo
private IColumnValuesReader createReader(PrimitiveSchemaNode
primitiveNode, List<IColumnValuesReader> readers) {
IColumnValuesReader reader;
if (delimiters.isEmpty()) {
- reader =
readerFactory.createValueReader(primitiveNode.getTypeTag(),
primitiveNode.getColumnIndex(), level,
- primitiveNode.isPrimaryKey());
+ int nodeLevel = primitiveNode.isPrimaryKey() ? 1 : level;
+ reader =
readerFactory.createValueReader(primitiveNode.getTypeTag(),
primitiveNode.getColumnIndex(),
+ nodeLevel, primitiveNode.isPrimaryKey());
} else {
// array
reader =
readerFactory.createValueReader(primitiveNode.getTypeTag(),
primitiveNode.getColumnIndex(), level,
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/ColumnAssembler.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/ColumnAssembler.java
index f5a327deee..835bb50f63 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/ColumnAssembler.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/ColumnAssembler.java
@@ -110,6 +110,10 @@ public final class ColumnAssembler {
}
public int skip(int count) throws HyracksDataException {
+ if (numberOfTuples == 0) {
+ // Avoid advancing tupleIndex and numberOfSkips if a mega leaf
node is filtered out
+ return 0;
+ }
numberOfSkips += count;
tupleIndex += count;
for (int i = 0; i < assemblers.length; i++) {
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnMetadata.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnMetadata.java
index e32460978c..d50af3808c 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnMetadata.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnMetadata.java
@@ -33,6 +33,7 @@ import
org.apache.asterix.column.filter.FalseColumnFilterEvaluator;
import org.apache.asterix.column.filter.FilterAccessorProvider;
import org.apache.asterix.column.filter.IColumnFilterEvaluator;
import org.apache.asterix.column.filter.TrueColumnFilterEvaluator;
+import org.apache.asterix.column.filter.iterable.ColumnFilterEvaluatorContext;
import
org.apache.asterix.column.filter.iterable.IColumnIterableFilterEvaluator;
import
org.apache.asterix.column.filter.iterable.IColumnIterableFilterEvaluatorFactory;
import
org.apache.asterix.column.filter.range.IColumnRangeFilterEvaluatorFactory;
@@ -48,14 +49,11 @@ import
org.apache.asterix.column.values.IColumnValuesReaderFactory;
import org.apache.asterix.column.values.reader.PrimitiveColumnValuesReader;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.runtime.projection.FunctionCallInformation;
-import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
-import org.apache.hyracks.algebricks.runtime.evaluators.EvaluatorContext;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.IWarningCollector;
import org.apache.hyracks.data.std.api.IValueReference;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
-import org.apache.hyracks.dataflow.common.utils.TaskUtil;
import
org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleReader;
import org.apache.hyracks.util.LogRedactionUtil;
import org.apache.logging.log4j.LogManager;
@@ -207,20 +205,18 @@ public class QueryColumnMetadata extends
AbstractColumnImmutableReadMetadata {
List<IColumnRangeFilterValueAccessor> filterValueAccessors =
Collections.emptyList();
String jobId = null;
if (context != null) {
- FilterAccessorProvider filterAccessorProvider =
+ FilterAccessorProvider accessorProvider =
new FilterAccessorProvider(root, clipperVisitor,
readerFactory, valueGetterFactory);
- TaskUtil.put(FilterAccessorProvider.FILTER_ACCESSOR_PROVIDER_KEY,
filterAccessorProvider, context);
// Min/Max filters in page0
- normalizedFilterEvaluator =
normalizedEvaluatorFactory.create(filterAccessorProvider);
- filterValueAccessors = filterAccessorProvider.getFilterAccessors();
-
+ normalizedFilterEvaluator =
normalizedEvaluatorFactory.create(accessorProvider);
+ filterValueAccessors = accessorProvider.getFilterAccessors();
// Filter columns (columns appeared in WHERE clause)
- IEvaluatorContext evaluatorContext = new EvaluatorContext(context);
+ ColumnFilterEvaluatorContext evaluatorContext = new
ColumnFilterEvaluatorContext(context, accessorProvider);
// ignore atomic (or flat) types information
clipperVisitor.setIgnoreFlatType(true);
- filterAccessorProvider.reset();
- columnFilterEvaluator =
columnFilterEvaluatorFactory.create(filterAccessorProvider, evaluatorContext);
- filterColumnReaders =
filterAccessorProvider.getFilterColumnReaders();
+ accessorProvider.reset();
+ columnFilterEvaluator =
columnFilterEvaluatorFactory.create(evaluatorContext);
+ filterColumnReaders = accessorProvider.getFilterColumnReaders();
jobId = context.getJobletContext().getJobId().toString();
}
// log normalized filter
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnWithMetaMetadata.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnWithMetaMetadata.java
index b4de820169..34e40ffa3d 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnWithMetaMetadata.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnWithMetaMetadata.java
@@ -30,6 +30,7 @@ import
org.apache.asterix.column.assembler.value.IValueGetterFactory;
import org.apache.asterix.column.filter.FilterAccessorProvider;
import org.apache.asterix.column.filter.IColumnFilterEvaluator;
import org.apache.asterix.column.filter.TrueColumnFilterEvaluator;
+import org.apache.asterix.column.filter.iterable.ColumnFilterEvaluatorContext;
import
org.apache.asterix.column.filter.iterable.IColumnIterableFilterEvaluator;
import
org.apache.asterix.column.filter.iterable.IColumnIterableFilterEvaluatorFactory;
import
org.apache.asterix.column.filter.range.IColumnRangeFilterEvaluatorFactory;
@@ -44,14 +45,11 @@ import
org.apache.asterix.column.values.IColumnValuesReaderFactory;
import org.apache.asterix.column.values.reader.PrimitiveColumnValuesReader;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.runtime.projection.FunctionCallInformation;
-import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
-import org.apache.hyracks.algebricks.runtime.evaluators.EvaluatorContext;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.IWarningCollector;
import org.apache.hyracks.data.std.api.IValueReference;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
-import org.apache.hyracks.dataflow.common.utils.TaskUtil;
import
org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleReader;
/**
@@ -150,22 +148,21 @@ public final class QueryColumnWithMetaMetadata extends
QueryColumnMetadata {
IColumnIterableFilterEvaluator columnFilterEvaluator =
TrueColumnFilterEvaluator.INSTANCE;
List<IColumnValuesReader> filterColumnReaders =
Collections.emptyList();
List<IColumnRangeFilterValueAccessor> filterValueAccessors =
Collections.emptyList();
- String jobId = "";
+ String jobId = null;
if (context != null) {
- FilterAccessorProvider filterAccessorProvider =
+ FilterAccessorProvider accessorProvider =
new FilterAccessorProvider(root, clipperVisitor,
readerFactory, valueGetterFactory);
- TaskUtil.put(FilterAccessorProvider.FILTER_ACCESSOR_PROVIDER_KEY,
filterAccessorProvider, context);
// Min/Max filters in page0
- normalizedFilterEvaluator =
normalizedEvaluatorFactory.create(filterAccessorProvider);
- filterValueAccessors = filterAccessorProvider.getFilterAccessors();
+ normalizedFilterEvaluator =
normalizedEvaluatorFactory.create(accessorProvider);
+ filterValueAccessors = accessorProvider.getFilterAccessors();
// Filter columns (columns appeared in WHERE clause)
- IEvaluatorContext evaluatorContext = new EvaluatorContext(context);
+ ColumnFilterEvaluatorContext evaluatorContext = new
ColumnFilterEvaluatorContext(context, accessorProvider);
// ignore atomic (or flat) types information
clipperVisitor.setIgnoreFlatType(true);
- filterAccessorProvider.reset();
- columnFilterEvaluator =
columnFilterEvaluatorFactory.create(filterAccessorProvider, evaluatorContext);
- filterColumnReaders =
filterAccessorProvider.getFilterColumnReaders();
+ accessorProvider.reset();
+ columnFilterEvaluator =
columnFilterEvaluatorFactory.create(evaluatorContext);
+ filterColumnReaders = accessorProvider.getFilterColumnReaders();
jobId = context.getJobletContext().getJobId().toString();
}
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/value/key/FloatKeyValueReader.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/value/key/FloatKeyValueReader.java
index be83385f76..8040594227 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/value/key/FloatKeyValueReader.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/value/key/FloatKeyValueReader.java
@@ -34,7 +34,7 @@ public final class FloatKeyValueReader extends
AbstractFixedLengthColumnKeyValue
}
@Override
- public double getDouble() {
+ public float getFloat() {
return FloatPointable.getFloat(value.getByteArray(),
value.getStartOffset());
}
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/PushdownUtil.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/PushdownUtil.java
index aaaeea1bd8..86fa2bfe31 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/PushdownUtil.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/PushdownUtil.java
@@ -37,7 +37,10 @@ import org.apache.asterix.om.utils.ConstantExpressionUtil;
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import
org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
import
org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
import
org.apache.hyracks.algebricks.core.algebra.expressions.IAlgebricksConstantValue;
@@ -60,6 +63,27 @@ public class PushdownUtil {
private PushdownUtil() {
}
+ public static IVariableTypeEnvironment getTypeEnv(ILogicalOperator
useOperator, IOptimizationContext context)
+ throws AlgebricksException {
+ if (useOperator.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN
+ || useOperator.getOperatorTag() ==
LogicalOperatorTag.INNERJOIN) {
+ // Special case: for pushed select condition
+ return useOperator.computeOutputTypeEnvironment(context);
+ } else {
+ return useOperator.computeInputTypeEnvironment(context);
+ }
+ }
+
+ public static IVariableTypeEnvironment getTypeEnv(ILogicalOperator
useOperator, ILogicalOperator scanOperator,
+ IOptimizationContext context) throws AlgebricksException {
+ if (useOperator == scanOperator) {
+ // Special case: for pushed select condition
+ return useOperator.computeOutputTypeEnvironment(context);
+ } else {
+ return scanOperator.computeOutputTypeEnvironment(context);
+ }
+ }
+
public static String getFieldName(AbstractFunctionCallExpression
fieldAccessExpr, IVariableTypeEnvironment typeEnv)
throws AlgebricksException {
if
(BuiltinFunctions.FIELD_ACCESS_BY_NAME.equals(fieldAccessExpr.getFunctionIdentifier()))
{
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/AbstractColumnBTreeLeafFrame.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/AbstractColumnBTreeLeafFrame.java
index 9aeafa431c..6ecd59e33d 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/AbstractColumnBTreeLeafFrame.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/AbstractColumnBTreeLeafFrame.java
@@ -94,7 +94,8 @@ public abstract class AbstractColumnBTreeLeafFrame implements
ITreeIndexFrame {
@Override
public final void setPage(ICachedPage page) {
this.page = page;
- this.buf = page.getBuffer();
+ // Duplicate to avoid interference when scanning the dataset twice
+ this.buf = page.getBuffer().duplicate();
buf.clear();
buf.position(HEADER_SIZE);
}
