This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 903f0a15da219855310d2f55b4f8f86c9afd3b0e Author: Wail Alkowaileet <[email protected]> AuthorDate: Wed Aug 7 09:36:19 2024 +0300 [ASTERIXDB-3395][COMP] Resolve common expected schema nodes - user model changes: no - storage format changes: no - interface changes: yes Details: If a common node is used in multiple expressions and those expressions assigned to multiple variables, their parent nodes could be changed by one of those variables. When replacing a node, we won't be able to find its previous version. This fix uses the expression instead of nodes' references to find previously created nodes. Ext-ref: MB-61604 Change-Id: I3491632ba51231eb176ea70cb15ae31b611798df Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18624 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Wail Alkowaileet <[email protected]> Reviewed-by: Ali Alsuliman <[email protected]> --- .../schema/AbstractComplexExpectedSchemaNode.java | 29 ++++++------- .../schema/AbstractExpectedSchemaNode.java | 32 ++++++++++---- .../pushdown/schema/AnyExpectedSchemaNode.java | 24 ++++++----- .../pushdown/schema/ArrayExpectedSchemaNode.java | 9 ++-- .../pushdown/schema/ExpectedSchemaBuilder.java | 24 +++++------ .../rules/pushdown/schema/IExpectedSchemaNode.java | 25 +++++++---- .../pushdown/schema/ObjectExpectedSchemaNode.java | 49 +++++++++++++--------- .../pushdown/schema/RootExpectedSchemaNode.java | 5 ++- .../pushdown/schema/UnionExpectedSchemaNode.java | 28 ++++++++----- .../visitor/ExpectedSchemaMergerVisitor.java | 20 +++++---- .../ExpressionToExpectedSchemaNodeVisitor.java | 6 +-- .../other-pushdowns.024.query.sqlpp | 28 +++++++++++++ .../other-pushdowns.025.query.sqlpp | 27 ++++++++++++ .../other-pushdowns.026.query.sqlpp | 28 +++++++++++++ .../other-pushdowns.027.query.sqlpp | 30 +++++++++++++ .../pushdown-plans/pushdown-plans.12.query.sqlpp | 28 +++++++++++++ .../pushdown-plans/pushdown-plans.13.query.sqlpp | 27 ++++++++++++ .../pushdown-plans/pushdown-plans.14.query.sqlpp | 28 +++++++++++++ .../pushdown-plans/pushdown-plans.15.query.sqlpp | 30 +++++++++++++ .../other-pushdowns/other-pushdowns.024.plan | 38 +++++++++++++++++ .../other-pushdowns/other-pushdowns.025.plan | 22 ++++++++++ .../other-pushdowns/other-pushdowns.026.plan | 38 +++++++++++++++++ .../other-pushdowns/other-pushdowns.027.plan | 26 ++++++++++++ .../parquet/pushdown-plans/pushdown-plans.12.plan | 36 ++++++++++++++++ .../parquet/pushdown-plans/pushdown-plans.13.plan | 20 +++++++++ .../parquet/pushdown-plans/pushdown-plans.14.plan | 36 ++++++++++++++++ .../parquet/pushdown-plans/pushdown-plans.15.plan | 24 +++++++++++ .../other-pushdowns/other-pushdowns.024.plan | 38 +++++++++++++++++ .../other-pushdowns/other-pushdowns.025.plan | 22 ++++++++++ .../other-pushdowns/other-pushdowns.026.plan | 38 +++++++++++++++++ .../other-pushdowns/other-pushdowns.027.plan | 26 ++++++++++++ .../asterix/metadata/utils/PushdownUtil.java | 21 ++++++++++ 32 files changed, 761 insertions(+), 101 deletions(-) diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/AbstractComplexExpectedSchemaNode.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/AbstractComplexExpectedSchemaNode.java index 4ca0dd8e97..34ff582087 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/AbstractComplexExpectedSchemaNode.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/AbstractComplexExpectedSchemaNode.java @@ -18,13 +18,14 @@ */ package org.apache.asterix.optimizer.rules.pushdown.schema; -import org.apache.hyracks.api.exceptions.SourceLocation; +import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; +import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression; public abstract class AbstractComplexExpectedSchemaNode extends AbstractExpectedSchemaNode { - AbstractComplexExpectedSchemaNode(AbstractComplexExpectedSchemaNode parent, SourceLocation sourceLocation, - String functionName) { - super(parent, sourceLocation, functionName); + AbstractComplexExpectedSchemaNode(AbstractComplexExpectedSchemaNode parent, + AbstractFunctionCallExpression parentExpression, ILogicalExpression expression) { + super(parent, parentExpression, expression); } @Override @@ -33,8 +34,8 @@ public abstract class AbstractComplexExpectedSchemaNode extends AbstractExpected } @Override - public IExpectedSchemaNode replaceIfNeeded(ExpectedSchemaNodeType expectedNodeType, SourceLocation sourceLocation, - String functionName) { + public IExpectedSchemaNode replaceIfNeeded(ExpectedSchemaNodeType expectedNodeType, + AbstractFunctionCallExpression parentExpression, ILogicalExpression expression) { //If no change is required, return the same node IExpectedSchemaNode node = this; if (expectedNodeType == ExpectedSchemaNodeType.ANY) { @@ -47,7 +48,7 @@ public abstract class AbstractComplexExpectedSchemaNode extends AbstractExpected * In this case, we first saw (t.hashtags[*].text), but the next expression (t.hashtags) requested * the entire hashtags. So, the expected type for hashtags should be ANY */ - node = new AnyExpectedSchemaNode(getParent(), getSourceLocation(), getFunctionName(), false); + node = new AnyExpectedSchemaNode(getParent(), getParentExpression(), false); getParent().replaceChild(this, node); } else if (expectedNodeType != getType()) { /* @@ -56,8 +57,7 @@ public abstract class AbstractComplexExpectedSchemaNode extends AbstractExpected */ //Create UNION node and its parent is the parent of this - UnionExpectedSchemaNode unionSchemaNode = - new UnionExpectedSchemaNode(getParent(), getSourceLocation(), getFunctionName()); + UnionExpectedSchemaNode unionSchemaNode = new UnionExpectedSchemaNode(getParent(), getParentExpression()); //Add this as a child of UNION unionSchemaNode.addChild(this); @@ -78,7 +78,7 @@ public abstract class AbstractComplexExpectedSchemaNode extends AbstractExpected * Before: UNION <-- this * After: UNION <-- (this, newChild) */ - unionSchemaNode.createChild(expectedNodeType, sourceLocation, functionName); + unionSchemaNode.createChild(expectedNodeType, parentExpression, expression); node = unionSchemaNode; } return node; @@ -113,14 +113,15 @@ public abstract class AbstractComplexExpectedSchemaNode extends AbstractExpected } public static AbstractComplexExpectedSchemaNode createNestedNode(ExpectedSchemaNodeType type, - AbstractComplexExpectedSchemaNode parent, SourceLocation sourceLocation, String functionName) { + AbstractComplexExpectedSchemaNode parent, AbstractFunctionCallExpression parentExpression, + ILogicalExpression myExpr) { switch (type) { case ARRAY: - return new ArrayExpectedSchemaNode(parent, sourceLocation, functionName); + return new ArrayExpectedSchemaNode(parent, parentExpression, myExpr); case OBJECT: - return new ObjectExpectedSchemaNode(parent, sourceLocation, functionName); + return new ObjectExpectedSchemaNode(parent, parentExpression, myExpr); case UNION: - return new UnionExpectedSchemaNode(parent, sourceLocation, functionName); + return new UnionExpectedSchemaNode(parent, parentExpression); default: throw new IllegalStateException(type + " is not nested or unknown"); } diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/AbstractExpectedSchemaNode.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/AbstractExpectedSchemaNode.java index b895781374..379fea3494 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/AbstractExpectedSchemaNode.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/AbstractExpectedSchemaNode.java @@ -18,18 +18,21 @@ */ package org.apache.asterix.optimizer.rules.pushdown.schema; +import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; +import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag; +import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression; import org.apache.hyracks.api.exceptions.SourceLocation; abstract class AbstractExpectedSchemaNode implements IExpectedSchemaNode { + protected final AbstractFunctionCallExpression parentExpression; + protected final ILogicalExpression expression; private AbstractComplexExpectedSchemaNode parent; - private final SourceLocation sourceLocation; - private final String functionName; - AbstractExpectedSchemaNode(AbstractComplexExpectedSchemaNode parent, SourceLocation sourceLocation, - String functionName) { + AbstractExpectedSchemaNode(AbstractComplexExpectedSchemaNode parent, + AbstractFunctionCallExpression parentExpression, ILogicalExpression expression) { + this.parentExpression = parentExpression; + this.expression = expression; this.parent = parent; - this.sourceLocation = sourceLocation; - this.functionName = functionName; } @Override @@ -39,12 +42,25 @@ abstract class AbstractExpectedSchemaNode implements IExpectedSchemaNode { @Override public final SourceLocation getSourceLocation() { - return sourceLocation; + return expression.getSourceLocation(); } @Override public final String getFunctionName() { - return functionName; + if (expression.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) { + return ((AbstractFunctionCallExpression) expression).getFunctionIdentifier().getName(); + } + return null; + } + + @Override + public AbstractFunctionCallExpression getParentExpression() { + return parentExpression; + } + + @Override + public ILogicalExpression getExpression() { + return expression; } @Override diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/AnyExpectedSchemaNode.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/AnyExpectedSchemaNode.java index 80069e39a0..fc3046baed 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/AnyExpectedSchemaNode.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/AnyExpectedSchemaNode.java @@ -18,20 +18,22 @@ */ package org.apache.asterix.optimizer.rules.pushdown.schema; -import org.apache.hyracks.api.exceptions.SourceLocation; +import static org.apache.asterix.optimizer.rules.pushdown.schema.AbstractComplexExpectedSchemaNode.createNestedNode; + +import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; +import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression; public class AnyExpectedSchemaNode extends AbstractExpectedSchemaNode { private boolean replaceable; - public AnyExpectedSchemaNode(AbstractComplexExpectedSchemaNode parent, SourceLocation sourceLocation, - String functionName) { - super(parent, sourceLocation, functionName); + public AnyExpectedSchemaNode(AbstractComplexExpectedSchemaNode parent, AbstractFunctionCallExpression expression) { + super(parent, expression, expression); replaceable = true; } - protected AnyExpectedSchemaNode(AbstractComplexExpectedSchemaNode parent, SourceLocation sourceLocation, - String functionName, boolean replaceable) { - super(parent, sourceLocation, functionName); + protected AnyExpectedSchemaNode(AbstractComplexExpectedSchemaNode parent, AbstractFunctionCallExpression expression, + boolean replaceable) { + super(parent, expression, expression); this.replaceable = replaceable; } @@ -45,8 +47,8 @@ public class AnyExpectedSchemaNode extends AbstractExpectedSchemaNode { } @Override - public IExpectedSchemaNode replaceIfNeeded(ExpectedSchemaNodeType expectedNodeType, SourceLocation sourceLocation, - String functionName) { + public IExpectedSchemaNode replaceIfNeeded(ExpectedSchemaNodeType expectedNodeType, + AbstractFunctionCallExpression parentExpression, ILogicalExpression expression) { if (expectedNodeType == ExpectedSchemaNodeType.ANY) { return this; } @@ -57,8 +59,8 @@ public class AnyExpectedSchemaNode extends AbstractExpectedSchemaNode { * the given nested type. */ AbstractComplexExpectedSchemaNode parent = getParent(); - AbstractComplexExpectedSchemaNode nestedNode = AbstractComplexExpectedSchemaNode - .createNestedNode(expectedNodeType, parent, getSourceLocation(), functionName); + AbstractComplexExpectedSchemaNode nestedNode = + createNestedNode(expectedNodeType, parent, this.parentExpression, expression); return parent.replaceChild(this, nestedNode); } diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/ArrayExpectedSchemaNode.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/ArrayExpectedSchemaNode.java index e6478625f2..b8135b72e3 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/ArrayExpectedSchemaNode.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/ArrayExpectedSchemaNode.java @@ -18,14 +18,15 @@ */ package org.apache.asterix.optimizer.rules.pushdown.schema; -import org.apache.hyracks.api.exceptions.SourceLocation; +import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; +import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression; public class ArrayExpectedSchemaNode extends AbstractComplexExpectedSchemaNode { private IExpectedSchemaNode child; - public ArrayExpectedSchemaNode(AbstractComplexExpectedSchemaNode parent, SourceLocation sourceLocation, - String functionName) { - super(parent, sourceLocation, functionName); + public ArrayExpectedSchemaNode(AbstractComplexExpectedSchemaNode parent, + AbstractFunctionCallExpression parentExpression, ILogicalExpression expression) { + super(parent, parentExpression, expression); } @Override diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/ExpectedSchemaBuilder.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/ExpectedSchemaBuilder.java index d42ffedbad..e442d64ae9 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/ExpectedSchemaBuilder.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/ExpectedSchemaBuilder.java @@ -59,8 +59,7 @@ public class ExpectedSchemaBuilder { //Parent always nested AbstractComplexExpectedSchemaNode parent = (AbstractComplexExpectedSchemaNode) buildNestedNode(expr, typeEnv); if (parent != null) { - IExpectedSchemaNode leaf = - new AnyExpectedSchemaNode(parent, expr.getSourceLocation(), expr.getFunctionIdentifier().getName()); + IExpectedSchemaNode leaf = new AnyExpectedSchemaNode(parent, expr); IExpectedSchemaNode actualNode = addOrReplaceChild(expr, typeEnv, parent, leaf); if (producedVar != null) { //Register the node if a variable is produced @@ -84,8 +83,9 @@ public class ExpectedSchemaBuilder { varToNode.put(variable, RootExpectedSchemaNode.ALL_FIELDS_ROOT_IRREPLACEABLE_NODE); } else { // If it is a nested node, replace it to a LEAF node - AnyExpectedSchemaNode leafNode = (AnyExpectedSchemaNode) node.replaceIfNeeded(ExpectedSchemaNodeType.ANY, - parent.getSourceLocation(), parent.getFunctionName()); + // Both expressions are null as they're the node isn't used anymore and the node ANY is not replaceable + AnyExpectedSchemaNode leafNode = + (AnyExpectedSchemaNode) node.replaceIfNeeded(ExpectedSchemaNodeType.ANY, null, null); // make the leaf node irreplaceable leafNode.preventReplacing(); varToNode.put(variable, leafNode); @@ -118,7 +118,7 @@ public class ExpectedSchemaBuilder { if (isVariable(parentExpr)) { //A variable could be the record's originated from data-scan or an expression from assign LogicalVariable sourceVar = VariableUtilities.getVariable(parentExpr); - return changeNodeForVariable(sourceVar, myExpr); + return changeNodeForVariable(sourceVar, myExpr, myExpr); } //Recursively create the parent nodes. Parent is always a nested node @@ -134,8 +134,8 @@ public class ExpectedSchemaBuilder { * Create 'myNode'. It is a nested node because the function is either getField() or a supported array * function */ - AbstractComplexExpectedSchemaNode myNode = AbstractComplexExpectedSchemaNode.createNestedNode(myType, - newParent, myExpr.getSourceLocation(), myExpr.getFunctionIdentifier().getName()); + AbstractComplexExpectedSchemaNode myNode = + AbstractComplexExpectedSchemaNode.createNestedNode(myType, newParent, parentFuncExpr, myExpr); // Add (or replace old child with) myNode to the parent return addOrReplaceChild(parentFuncExpr, typeEnv, newParent, myNode); } @@ -148,7 +148,7 @@ public class ExpectedSchemaBuilder { } private IExpectedSchemaNode changeNodeForVariable(LogicalVariable sourceVar, - AbstractFunctionCallExpression myExpr) { + AbstractFunctionCallExpression parentExpression, ILogicalExpression expression) { //Get the associated node with the sourceVar (if any) IExpectedSchemaNode oldNode = varToNode.get(sourceVar); if (oldNode == null || !oldNode.allowsReplacing()) { @@ -157,10 +157,9 @@ public class ExpectedSchemaBuilder { return null; } //What is the expected type of the variable - ExpectedSchemaNodeType varExpectedType = getExpectedNestedNodeType(myExpr); + ExpectedSchemaNodeType varExpectedType = getExpectedNestedNodeType(parentExpression); // Get the node associated with the variable (or change its type if needed). - IExpectedSchemaNode newNode = oldNode.replaceIfNeeded(varExpectedType, myExpr.getSourceLocation(), - myExpr.getFunctionIdentifier().getName()); + IExpectedSchemaNode newNode = oldNode.replaceIfNeeded(varExpectedType, parentExpression, expression); //Map the sourceVar to the node varToNode.put(sourceVar, newNode); return newNode; @@ -195,11 +194,12 @@ public class ExpectedSchemaBuilder { private static IExpectedSchemaNode handleObject(AbstractFunctionCallExpression parentExpr, IVariableTypeEnvironment typeEnv, AbstractComplexExpectedSchemaNode parent, IExpectedSchemaNode child) throws AlgebricksException { + int fieldNameId = PushdownUtil.getFieldNameId(parentExpr); String fieldName = PushdownUtil.getFieldName(parentExpr, typeEnv); ObjectExpectedSchemaNode objectNode = (ObjectExpectedSchemaNode) parent; IExpectedSchemaNode actualChild = objectNode.getChildren().get(fieldName); if (actualChild == null) { - objectNode.addChild(fieldName, child); + objectNode.addChild(fieldName, fieldNameId, child); actualChild = child; } else { actualChild = objectNode.replaceChild(actualChild, child); diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/IExpectedSchemaNode.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/IExpectedSchemaNode.java index 345cb84b5e..decafec442 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/IExpectedSchemaNode.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/IExpectedSchemaNode.java @@ -18,7 +18,8 @@ */ package org.apache.asterix.optimizer.rules.pushdown.schema; -import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; +import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; +import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression; import org.apache.hyracks.api.exceptions.SourceLocation; /** @@ -41,11 +42,21 @@ public interface IExpectedSchemaNode { */ String getFunctionName(); + /** + * @return function expression of which determined the type of the parent node + */ + AbstractFunctionCallExpression getParentExpression(); + /** * @return the parent of a node */ AbstractComplexExpectedSchemaNode getParent(); + /** + * @return this node's expression + */ + ILogicalExpression getExpression(); + /** * Set parent of a node * @@ -75,11 +86,11 @@ public interface IExpectedSchemaNode { * - {@link ExpectedSchemaNodeType#OBJECT} to {@link ExpectedSchemaNodeType#UNION} * * @param expectedNodeType what is the other expected type - * @param sourceLocation source location of the value access - * @param functionName function name as in {@link FunctionIdentifier#getName()} - * @see AbstractComplexExpectedSchemaNode#replaceIfNeeded(ExpectedSchemaNodeType, SourceLocation, String) - * @see UnionExpectedSchemaNode#replaceIfNeeded(ExpectedSchemaNodeType, SourceLocation, String) + * @param parentExpression parent expression + * @param expression this node's expression + * @see IExpectedSchemaNode#replaceIfNeeded(ExpectedSchemaNodeType, AbstractFunctionCallExpression, ILogicalExpression) + * @see IExpectedSchemaNode#replaceIfNeeded(ExpectedSchemaNodeType, AbstractFunctionCallExpression, ILogicalExpression) */ - IExpectedSchemaNode replaceIfNeeded(ExpectedSchemaNodeType expectedNodeType, SourceLocation sourceLocation, - String functionName); + IExpectedSchemaNode replaceIfNeeded(ExpectedSchemaNodeType expectedNodeType, + AbstractFunctionCallExpression parentExpression, ILogicalExpression expression); } diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/ObjectExpectedSchemaNode.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/ObjectExpectedSchemaNode.java index a6d43994f4..e220437707 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/ObjectExpectedSchemaNode.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/ObjectExpectedSchemaNode.java @@ -21,27 +21,43 @@ package org.apache.asterix.optimizer.rules.pushdown.schema; import java.util.HashMap; import java.util.Map; -import org.apache.hyracks.api.exceptions.SourceLocation; +import org.apache.asterix.metadata.utils.PushdownUtil; +import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; +import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression; +import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; + +import it.unimi.dsi.fastutil.ints.Int2ObjectMap; +import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; public class ObjectExpectedSchemaNode extends AbstractComplexExpectedSchemaNode { private final Map<String, IExpectedSchemaNode> children; + private final Int2ObjectMap<String> fieldIdToFieldName; - public ObjectExpectedSchemaNode(AbstractComplexExpectedSchemaNode parent, SourceLocation sourceLocation, - String functionName) { - super(parent, sourceLocation, functionName); + public ObjectExpectedSchemaNode(AbstractComplexExpectedSchemaNode parent, + AbstractFunctionCallExpression parentExpression, ILogicalExpression expression) { + super(parent, parentExpression, expression); children = new HashMap<>(); + fieldIdToFieldName = new Int2ObjectOpenHashMap<>(); } public boolean isRoot() { return false; } - public Map<String, IExpectedSchemaNode> getChildren() { - return children; + public void addChild(String fieldName, int fieldId, IExpectedSchemaNode child) { + FunctionIdentifier fid = child.getParentExpression().getFunctionIdentifier(); + children.put(fieldName, child); + if (fieldId > -1) { + fieldIdToFieldName.put(fieldId, fieldName); + } } - public void addChild(String fieldName, IExpectedSchemaNode child) { - children.put(fieldName, child); + public void addAllFieldNameIds(ObjectExpectedSchemaNode node) { + fieldIdToFieldName.putAll(node.fieldIdToFieldName); + } + + public Map<String, IExpectedSchemaNode> getChildren() { + return children; } @Override @@ -71,18 +87,13 @@ public class ObjectExpectedSchemaNode extends AbstractComplexExpectedSchemaNode } public String getChildFieldName(IExpectedSchemaNode requestedChild) { - String key = null; - for (Map.Entry<String, IExpectedSchemaNode> child : children.entrySet()) { - if (child.getValue() == requestedChild) { - key = child.getKey(); - break; - } - } + AbstractFunctionCallExpression expr = requestedChild.getParentExpression(); + int fieldNameId = PushdownUtil.getFieldNameId(requestedChild.getParentExpression()); - if (key == null) { - //this should not happen - throw new IllegalStateException("Node " + requestedChild.getType() + " is not a child"); + if (fieldNameId > -1) { + return fieldIdToFieldName.get(fieldNameId); } - return key; + + return PushdownUtil.getFieldName(expr); } } diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/RootExpectedSchemaNode.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/RootExpectedSchemaNode.java index 72c5c52ffb..e72e9971f1 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/RootExpectedSchemaNode.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/RootExpectedSchemaNode.java @@ -18,7 +18,8 @@ */ package org.apache.asterix.optimizer.rules.pushdown.schema; -import org.apache.hyracks.api.exceptions.SourceLocation; +import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; +import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression; public class RootExpectedSchemaNode extends ObjectExpectedSchemaNode { //Root with zero fields @@ -56,7 +57,7 @@ public class RootExpectedSchemaNode extends ObjectExpectedSchemaNode { @Override public AbstractComplexExpectedSchemaNode replaceIfNeeded(ExpectedSchemaNodeType expectedNodeType, - SourceLocation sourceLocation, String functionName) { + AbstractFunctionCallExpression parentExpression, ILogicalExpression expression) { if (rootType == ALL_FIELDS_ROOT) { //ALL_FIELDS_ROOT. Return a new CLIPPED_ROOT root return new RootExpectedSchemaNode(); diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/UnionExpectedSchemaNode.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/UnionExpectedSchemaNode.java index af6d2be714..a15e359464 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/UnionExpectedSchemaNode.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/schema/UnionExpectedSchemaNode.java @@ -22,14 +22,15 @@ import java.util.EnumMap; import java.util.Map; import java.util.Set; -import org.apache.hyracks.api.exceptions.SourceLocation; +import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; +import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression; public class UnionExpectedSchemaNode extends AbstractComplexExpectedSchemaNode { private final Map<ExpectedSchemaNodeType, AbstractComplexExpectedSchemaNode> children; - public UnionExpectedSchemaNode(AbstractComplexExpectedSchemaNode parent, SourceLocation sourceLocation, - String functionName) { - super(parent, sourceLocation, functionName); + public UnionExpectedSchemaNode(AbstractComplexExpectedSchemaNode parent, + AbstractFunctionCallExpression expression) { + super(parent, expression, expression); children = new EnumMap<>(ExpectedSchemaNodeType.class); } @@ -46,8 +47,13 @@ public class UnionExpectedSchemaNode extends AbstractComplexExpectedSchemaNode { children.put(node.getType(), node); } - public void createChild(ExpectedSchemaNodeType nodeType, SourceLocation sourceLocation, String functionName) { - children.computeIfAbsent(nodeType, k -> createNestedNode(k, this, sourceLocation, functionName)); + public void createChild(ExpectedSchemaNodeType nodeType, AbstractFunctionCallExpression parentExpression, + ILogicalExpression expression) { + if (parentExpression == null) { + // Should never happen + throw new NullPointerException("expression is null"); + } + children.computeIfAbsent(nodeType, k -> createNestedNode(k, this, parentExpression, expression)); } public AbstractComplexExpectedSchemaNode getChild(ExpectedSchemaNodeType type) { @@ -73,15 +79,15 @@ public class UnionExpectedSchemaNode extends AbstractComplexExpectedSchemaNode { * UNION type - we simply return this. In case we want to fallback to ANY node, we call the super method. * * @param expectedNodeType the expected type - * @param sourceLocation source location of the value access - * @param functionName function name of the expression + * @param parentExpression + * @param expression * @return ANY or this */ @Override - public IExpectedSchemaNode replaceIfNeeded(ExpectedSchemaNodeType expectedNodeType, SourceLocation sourceLocation, - String functionName) { + public IExpectedSchemaNode replaceIfNeeded(ExpectedSchemaNodeType expectedNodeType, + AbstractFunctionCallExpression parentExpression, ILogicalExpression expression) { if (expectedNodeType == ExpectedSchemaNodeType.ANY) { - return super.replaceIfNeeded(expectedNodeType, sourceLocation, functionName); + return super.replaceIfNeeded(expectedNodeType, parentExpression, expression); } return this; } diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/visitor/ExpectedSchemaMergerVisitor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/visitor/ExpectedSchemaMergerVisitor.java index d9ab05206a..09046b2086 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/visitor/ExpectedSchemaMergerVisitor.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/visitor/ExpectedSchemaMergerVisitor.java @@ -70,19 +70,23 @@ public class ExpectedSchemaMergerVisitor // combine RootExpectedSchemaNode mergedRoot = (RootExpectedSchemaNode) RootExpectedSchemaNode.ALL_FIELDS_ROOT_NODE - .replaceIfNeeded(ExpectedSchemaNodeType.OBJECT, node.getSourceLocation(), node.getFunctionName()); + .replaceIfNeeded(ExpectedSchemaNodeType.OBJECT, null, null); mergeObjectFields(mergedRoot, node.getChildren(), argRoot.getChildren()); + mergedRoot.addAllFieldNameIds(node); + mergedRoot.addAllFieldNameIds(argRoot); return mergedRoot; } @Override public IExpectedSchemaNode visit(ObjectExpectedSchemaNode node, IExpectedSchemaNode arg) { ObjectExpectedSchemaNode mergedObject = - new ObjectExpectedSchemaNode(currentParent, node.getSourceLocation(), node.getFunctionName()); + new ObjectExpectedSchemaNode(currentParent, node.getParentExpression(), node.getExpression()); Map<String, IExpectedSchemaNode> argChildren = Collections.emptyMap(); + mergedObject.addAllFieldNameIds(node); if (arg != null) { ObjectExpectedSchemaNode argObject = (ObjectExpectedSchemaNode) arg; argChildren = argObject.getChildren(); + mergedObject.addAllFieldNameIds(argObject); } mergeObjectFields(mergedObject, node.getChildren(), argChildren); @@ -99,7 +103,7 @@ public class ExpectedSchemaMergerVisitor argItem = arrayArg.getChild(); } ArrayExpectedSchemaNode mergedArray = - new ArrayExpectedSchemaNode(currentParent, node.getSourceLocation(), node.getFunctionName()); + new ArrayExpectedSchemaNode(currentParent, node.getParentExpression(), node.getExpression()); AbstractComplexExpectedSchemaNode previousParent = currentParent; currentParent = mergedArray; IExpectedSchemaNode mergedItem = merge(nodeItem, argItem); @@ -111,8 +115,7 @@ public class ExpectedSchemaMergerVisitor @Override public IExpectedSchemaNode visit(UnionExpectedSchemaNode node, IExpectedSchemaNode arg) { - UnionExpectedSchemaNode union = - new UnionExpectedSchemaNode(currentParent, node.getSourceLocation(), node.getFunctionName()); + UnionExpectedSchemaNode union = new UnionExpectedSchemaNode(currentParent, node.getParentExpression()); AbstractComplexExpectedSchemaNode previousParent = currentParent; currentParent = union; @@ -148,7 +151,7 @@ public class ExpectedSchemaMergerVisitor @Override public IExpectedSchemaNode visit(AnyExpectedSchemaNode node, IExpectedSchemaNode arg) { - return new AnyExpectedSchemaNode(currentParent, node.getSourceLocation(), node.getFunctionName()); + return new AnyExpectedSchemaNode(currentParent, node.getParentExpression()); } private void mergeObjectFields(ObjectExpectedSchemaNode objectNode, Map<String, IExpectedSchemaNode> left, @@ -170,7 +173,7 @@ public class ExpectedSchemaMergerVisitor } IExpectedSchemaNode rightChild = right.get(fieldName); IExpectedSchemaNode mergedChild = merge(leftChild.getValue(), rightChild); - objectNode.addChild(fieldName, mergedChild); + objectNode.addChild(fieldName, -1, mergedChild); mergedFields.add(fieldName); } } @@ -196,8 +199,7 @@ public class ExpectedSchemaMergerVisitor } private IExpectedSchemaNode createUnionNode(IExpectedSchemaNode leftChild, IExpectedSchemaNode rightChild) { - UnionExpectedSchemaNode union = - new UnionExpectedSchemaNode(currentParent, leftChild.getSourceLocation(), leftChild.getFunctionName()); + UnionExpectedSchemaNode union = new UnionExpectedSchemaNode(currentParent, leftChild.getParentExpression()); AbstractComplexExpectedSchemaNode previousParent = currentParent; currentParent = union; // Create a copy of leftChild diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/visitor/ExpressionToExpectedSchemaNodeVisitor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/visitor/ExpressionToExpectedSchemaNodeVisitor.java index cccc9368d3..0ef81aa6b8 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/visitor/ExpressionToExpectedSchemaNodeVisitor.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/visitor/ExpressionToExpectedSchemaNodeVisitor.java @@ -111,8 +111,7 @@ public class ExpressionToExpectedSchemaNodeVisitor implements ILogicalExpression } AbstractComplexExpectedSchemaNode newParent = replaceIfNeeded(parent, expr); - IExpectedSchemaNode myNode = - new AnyExpectedSchemaNode(newParent, expr.getSourceLocation(), expr.getFunctionIdentifier().getName()); + IExpectedSchemaNode myNode = new AnyExpectedSchemaNode(newParent, expr); ExpectedSchemaBuilder.addOrReplaceChild(expr, typeEnv, newParent, myNode); return myNode; } @@ -120,7 +119,6 @@ public class ExpressionToExpectedSchemaNodeVisitor implements ILogicalExpression private AbstractComplexExpectedSchemaNode replaceIfNeeded(IExpectedSchemaNode parent, AbstractFunctionCallExpression funcExpr) { ExpectedSchemaNodeType expectedType = ExpectedSchemaBuilder.getExpectedNestedNodeType(funcExpr); - return (AbstractComplexExpectedSchemaNode) parent.replaceIfNeeded(expectedType, parent.getSourceLocation(), - parent.getFunctionName()); + return (AbstractComplexExpectedSchemaNode) parent.replaceIfNeeded(expectedType, funcExpr, funcExpr); } } diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/pushdown/other-pushdowns/other-pushdowns.024.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/pushdown/other-pushdowns/other-pushdowns.024.query.sqlpp new file mode 100644 index 0000000000..52075b9f14 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/pushdown/other-pushdowns/other-pushdowns.024.query.sqlpp @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; +SET `compiler.parallelism` "0"; +SET `compiler.sort.parallel` "false"; +EXPLAIN +SELECT c.f1[0].f2[0][0], + c.f1[1].f3[1], + c.f1[1].f2[1][1] +FROM ColumnDataset c +ORDER BY c.x diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/pushdown/other-pushdowns/other-pushdowns.025.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/pushdown/other-pushdowns/other-pushdowns.025.query.sqlpp new file mode 100644 index 0000000000..4ee3627f9f --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/pushdown/other-pushdowns/other-pushdowns.025.query.sqlpp @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; +SET `compiler.parallelism` "0"; +SET `compiler.sort.parallel` "false"; +EXPLAIN +SELECT c.f1[0].f2[0][0], + c.f1[1].f3[1], + c.f1[1].f2[1][1], c.x +FROM ColumnDataset c diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/pushdown/other-pushdowns/other-pushdowns.026.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/pushdown/other-pushdowns/other-pushdowns.026.query.sqlpp new file mode 100644 index 0000000000..c2acf29237 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/pushdown/other-pushdowns/other-pushdowns.026.query.sqlpp @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; +SET `compiler.parallelism` "0"; +SET `compiler.sort.parallel` "false"; +EXPLAIN +SELECT c.f1[0].f2[0][0], + c.f1[1].f3[1], + c.f1[1].f2[1][1], c.x +FROM ColumnDataset c +ORDER BY c.y diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/pushdown/other-pushdowns/other-pushdowns.027.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/pushdown/other-pushdowns/other-pushdowns.027.query.sqlpp new file mode 100644 index 0000000000..03ae71d269 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/pushdown/other-pushdowns/other-pushdowns.027.query.sqlpp @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; +SET `compiler.parallelism` "0"; +SET `compiler.sort.parallel` "false"; +EXPLAIN +SELECT c.f1[0].f2[0][0], + -- Access f3 as an array + c.f1[1].f3[1], + c.f1[1].f2[1][1], + -- Access f3 as an object + c.f1[1].f3.f4 +FROM ColumnDataset c diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.12.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.12.query.sqlpp new file mode 100644 index 0000000000..d2802d061b --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.12.query.sqlpp @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; +SET `compiler.parallelism` "0"; +SET `compiler.sort.parallel` "false"; +EXPLAIN +SELECT p.f1[0].f2[0][0], + p.f1[1].f3[1], + p.f1[1].f2[1][1] +FROM ParquetDataset1 p +ORDER BY p.x diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.13.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.13.query.sqlpp new file mode 100644 index 0000000000..49d16af4ba --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.13.query.sqlpp @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; +SET `compiler.parallelism` "0"; +SET `compiler.sort.parallel` "false"; +EXPLAIN +SELECT p.f1[0].f2[0][0], + p.f1[1].f3[1], + p.f1[1].f2[1][1], p.x +FROM ParquetDataset1 p diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.14.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.14.query.sqlpp new file mode 100644 index 0000000000..1aa9d9c3f6 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.14.query.sqlpp @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; +SET `compiler.parallelism` "0"; +SET `compiler.sort.parallel` "false"; +EXPLAIN +SELECT p.f1[0].f2[0][0], + p.f1[1].f3[1], + p.f1[1].f2[1][1], p.x +FROM ParquetDataset1 p +ORDER BY p.y diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.15.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.15.query.sqlpp new file mode 100644 index 0000000000..e8efb2585a --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.15.query.sqlpp @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; +SET `compiler.parallelism` "0"; +SET `compiler.sort.parallel` "false"; +EXPLAIN +SELECT p.f1[0].f2[0][0], + -- Access f3 as an array + p.f1[1].f3[1], + p.f1[1].f2[1][1], + -- Access f3 as an object + p.f1[1].f3.f4 +FROM ParquetDataset1 p diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/column/pushdown/other-pushdowns/other-pushdowns.024.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/pushdown/other-pushdowns/other-pushdowns.024.plan new file mode 100644 index 0000000000..cc4aea03c6 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/pushdown/other-pushdowns/other-pushdowns.024.plan @@ -0,0 +1,38 @@ +distribute result [$$31] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] +-- DISTRIBUTE_RESULT |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + project ([$$31]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$31] <- [{"$1": get-item(get-item($$36, 0), 0), "$2": get-item($$37, 1), "$3": get-item(get-item($$38, 1), 1)}] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |PARTITIONED| + project ([$$36, $$37, $$38]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- SORT_MERGE_EXCHANGE [$$35(ASC) ] |PARTITIONED| + order (ASC, $$35) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STABLE_SORT [$$35(ASC)] |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + project ([$$36, $$37, $$38, $$35]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$38, $$37] <- [$$49.getField("f2"), $$49.getField("f3")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |PARTITIONED| + project ([$$35, $$49, $$36]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$49, $$36] <- [get-item($$33, 1), get-item($$33, 0).getField("f2")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |PARTITIONED| + project ([$$33, $$35]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$33, $$35] <- [$$c.getField("f1"), $$c.getField("x")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |PARTITIONED| + project ([$$c]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + data-scan []<-[$$34, $$c] <- test.ColumnDataset project ({x:any,f1:[{f2:[[any]],f3:[any]}]}) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- DATASOURCE_SCAN |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/column/pushdown/other-pushdowns/other-pushdowns.025.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/pushdown/other-pushdowns/other-pushdowns.025.plan new file mode 100644 index 0000000000..830e096bd5 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/pushdown/other-pushdowns/other-pushdowns.025.plan @@ -0,0 +1,22 @@ +distribute result [$$32] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] +-- DISTRIBUTE_RESULT |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + project ([$$32]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$32] <- [{"$1": get-item(get-item(get-item($$33, 0).getField("f2"), 0), 0), "$2": get-item($$49.getField("f3"), 1), "$3": get-item(get-item($$49.getField("f2"), 1), 1), "x": $$c.getField("x")}] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |PARTITIONED| + assign [$$49] <- [get-item($$33, 1)] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |PARTITIONED| + assign [$$33] <- [$$c.getField("f1")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |PARTITIONED| + project ([$$c]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + data-scan []<-[$$34, $$c] <- test.ColumnDataset project ({x:any,f1:[{f2:[[any]],f3:[any]}]}) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- DATASOURCE_SCAN |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/column/pushdown/other-pushdowns/other-pushdowns.026.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/pushdown/other-pushdowns/other-pushdowns.026.plan new file mode 100644 index 0000000000..b65bd37805 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/pushdown/other-pushdowns/other-pushdowns.026.plan @@ -0,0 +1,38 @@ +distribute result [$$32] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] +-- DISTRIBUTE_RESULT |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + project ([$$32]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$32] <- [{"$1": get-item(get-item($$37, 0), 0), "$2": get-item($$38, 1), "$3": get-item(get-item($$39, 1), 1), "x": $$40}] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |PARTITIONED| + project ([$$37, $$38, $$39, $$40]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- SORT_MERGE_EXCHANGE [$$36(ASC) ] |PARTITIONED| + order (ASC, $$36) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STABLE_SORT [$$36(ASC)] |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + project ([$$37, $$38, $$39, $$40, $$36]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$39, $$38] <- [$$51.getField("f2"), $$51.getField("f3")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |PARTITIONED| + project ([$$36, $$40, $$51, $$37]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$51, $$37] <- [get-item($$34, 1), get-item($$34, 0).getField("f2")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |PARTITIONED| + project ([$$34, $$36, $$40]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$34, $$36, $$40] <- [$$c.getField("f1"), $$c.getField("y"), $$c.getField("x")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |PARTITIONED| + project ([$$c]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + data-scan []<-[$$35, $$c] <- test.ColumnDataset project ({x:any,y:any,f1:[{f2:[[any]],f3:[any]}]}) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- DATASOURCE_SCAN |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/column/pushdown/other-pushdowns/other-pushdowns.027.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/pushdown/other-pushdowns/other-pushdowns.027.plan new file mode 100644 index 0000000000..64d7359516 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/pushdown/other-pushdowns/other-pushdowns.027.plan @@ -0,0 +1,26 @@ +distribute result [$$35] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] +-- DISTRIBUTE_RESULT |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + project ([$$35]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$35] <- [{"$1": get-item(get-item(get-item($$36, 0).getField("f2"), 0), 0), "$2": get-item($$43, 1), "$3": get-item(get-item($$55.getField("f2"), 1), 1), "f4": $$43.getField("f4")}] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |PARTITIONED| + assign [$$43] <- [$$55.getField("f3")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |PARTITIONED| + assign [$$55] <- [get-item($$36, 1)] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |PARTITIONED| + project ([$$36]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$36] <- [$$c.getField("f1")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |PARTITIONED| + project ([$$c]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + data-scan []<-[$$37, $$c] <- test.ColumnDataset project ({f1:[{f2:[[any]],f3:<[any],{f4:any}>}]}) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- DATASOURCE_SCAN |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/pushdown-plans/pushdown-plans.12.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/pushdown-plans/pushdown-plans.12.plan new file mode 100644 index 0000000000..8b117a9d72 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/pushdown-plans/pushdown-plans.12.plan @@ -0,0 +1,36 @@ +distribute result [$$31] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] +-- DISTRIBUTE_RESULT |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + project ([$$31]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$31] <- [{"$1": get-item(get-item($$35, 0), 0), "$2": get-item($$36, 1), "$3": get-item(get-item($$37, 1), 1)}] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |PARTITIONED| + project ([$$35, $$36, $$37]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- SORT_MERGE_EXCHANGE [$$34(ASC) ] |PARTITIONED| + order (ASC, $$34) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STABLE_SORT [$$34(ASC)] |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + project ([$$35, $$36, $$37, $$34]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$37, $$36] <- [$$48.getField("f2"), $$48.getField("f3")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |PARTITIONED| + project ([$$34, $$48, $$35]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$48, $$35] <- [get-item($$33, 1), get-item($$33, 0).getField("f2")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |PARTITIONED| + project ([$$33, $$34]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$33, $$34] <- [$$p.getField("f1"), $$p.getField("x")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + data-scan []<-[$$p] <- test.ParquetDataset1 project ({x:any,f1:[{f2:[[any]],f3:[any]}]}) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- DATASOURCE_SCAN |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/pushdown-plans/pushdown-plans.13.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/pushdown-plans/pushdown-plans.13.plan new file mode 100644 index 0000000000..3266da345e --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/pushdown-plans/pushdown-plans.13.plan @@ -0,0 +1,20 @@ +distribute result [$$32] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] +-- DISTRIBUTE_RESULT |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + project ([$$32]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$32] <- [{"$1": get-item(get-item(get-item($$33, 0).getField("f2"), 0), 0), "$2": get-item($$48.getField("f3"), 1), "$3": get-item(get-item($$48.getField("f2"), 1), 1), "x": $$p.getField("x")}] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |PARTITIONED| + assign [$$48] <- [get-item($$33, 1)] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |PARTITIONED| + assign [$$33] <- [$$p.getField("f1")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + data-scan []<-[$$p] <- test.ParquetDataset1 project ({x:any,f1:[{f2:[[any]],f3:[any]}]}) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- DATASOURCE_SCAN |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/pushdown-plans/pushdown-plans.14.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/pushdown-plans/pushdown-plans.14.plan new file mode 100644 index 0000000000..2b10297090 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/pushdown-plans/pushdown-plans.14.plan @@ -0,0 +1,36 @@ +distribute result [$$32] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] +-- DISTRIBUTE_RESULT |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + project ([$$32]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$32] <- [{"$1": get-item(get-item($$36, 0), 0), "$2": get-item($$37, 1), "$3": get-item(get-item($$38, 1), 1), "x": $$39}] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |PARTITIONED| + project ([$$36, $$37, $$38, $$39]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- SORT_MERGE_EXCHANGE [$$35(ASC) ] |PARTITIONED| + order (ASC, $$35) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STABLE_SORT [$$35(ASC)] |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + project ([$$36, $$37, $$38, $$39, $$35]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$38, $$37] <- [$$50.getField("f2"), $$50.getField("f3")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |PARTITIONED| + project ([$$35, $$39, $$50, $$36]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$50, $$36] <- [get-item($$34, 1), get-item($$34, 0).getField("f2")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |PARTITIONED| + project ([$$34, $$35, $$39]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$34, $$35, $$39] <- [$$p.getField("f1"), $$p.getField("y"), $$p.getField("x")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + data-scan []<-[$$p] <- test.ParquetDataset1 project ({x:any,y:any,f1:[{f2:[[any]],f3:[any]}]}) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- DATASOURCE_SCAN |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/pushdown-plans/pushdown-plans.15.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/pushdown-plans/pushdown-plans.15.plan new file mode 100644 index 0000000000..c575b1bf9b --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/pushdown-plans/pushdown-plans.15.plan @@ -0,0 +1,24 @@ +distribute result [$$35] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] +-- DISTRIBUTE_RESULT |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + project ([$$35]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$35] <- [{"$1": get-item(get-item(get-item($$36, 0).getField("f2"), 0), 0), "$2": get-item($$42, 1), "$3": get-item(get-item($$54.getField("f2"), 1), 1), "f4": $$42.getField("f4")}] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |PARTITIONED| + assign [$$42] <- [$$54.getField("f3")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |PARTITIONED| + assign [$$54] <- [get-item($$36, 1)] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |PARTITIONED| + project ([$$36]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$36] <- [$$p.getField("f1")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + data-scan []<-[$$p] <- test.ParquetDataset1 project ({f1:[{f2:[[any]],f3:<[any],{f4:any}>}]}) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- DATASOURCE_SCAN |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/column/pushdown/other-pushdowns/other-pushdowns.024.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/column/pushdown/other-pushdowns/other-pushdowns.024.plan new file mode 100644 index 0000000000..fde7f81d85 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/column/pushdown/other-pushdowns/other-pushdowns.024.plan @@ -0,0 +1,38 @@ +distribute result [$$31] [cardinality: 2.0, op-cost: 0.0, total-cost: 4.0] +-- DISTRIBUTE_RESULT |PARTITIONED| + exchange [cardinality: 2.0, op-cost: 0.0, total-cost: 4.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + project ([$$31]) [cardinality: 2.0, op-cost: 0.0, total-cost: 4.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$31] <- [{"$1": get-item(get-item($$36, 0), 0), "$2": get-item($$37, 1), "$3": get-item(get-item($$38, 1), 1)}] [cardinality: 2.0, op-cost: 0.0, total-cost: 4.0] + -- ASSIGN |PARTITIONED| + project ([$$36, $$37, $$38]) [cardinality: 2.0, op-cost: 0.0, total-cost: 4.0] + -- STREAM_PROJECT |PARTITIONED| + exchange [cardinality: 2.0, op-cost: 0.0, total-cost: 4.0] + -- SORT_MERGE_EXCHANGE [$$35(ASC) ] |PARTITIONED| + order (ASC, $$35) [cardinality: 2.0, op-cost: 2.0, total-cost: 4.0] + -- STABLE_SORT [$$35(ASC)] |PARTITIONED| + exchange [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + project ([$$36, $$37, $$38, $$35]) [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$38, $$37] <- [$$49.getField("f2"), $$49.getField("f3")] [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] + -- ASSIGN |PARTITIONED| + project ([$$35, $$49, $$36]) [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$49, $$36] <- [get-item($$33, 1), get-item($$33, 0).getField("f2")] [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] + -- ASSIGN |PARTITIONED| + project ([$$33, $$35]) [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$33, $$35] <- [$$c.getField("f1"), $$c.getField("x")] [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] + -- ASSIGN |PARTITIONED| + project ([$$c]) [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] + -- STREAM_PROJECT |PARTITIONED| + exchange [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + data-scan []<-[$$34, $$c] <- test.ColumnDataset project ({x:any,f1:[{f2:[[any]],f3:[any]}]}) [cardinality: 2.0, op-cost: 2.0, total-cost: 2.0] + -- DATASOURCE_SCAN |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/column/pushdown/other-pushdowns/other-pushdowns.025.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/column/pushdown/other-pushdowns/other-pushdowns.025.plan new file mode 100644 index 0000000000..deeb1ebb05 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/column/pushdown/other-pushdowns/other-pushdowns.025.plan @@ -0,0 +1,22 @@ +distribute result [$$32] [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] +-- DISTRIBUTE_RESULT |PARTITIONED| + exchange [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + project ([$$32]) [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$32] <- [{"$1": get-item(get-item(get-item($$33, 0).getField("f2"), 0), 0), "$2": get-item($$49.getField("f3"), 1), "$3": get-item(get-item($$49.getField("f2"), 1), 1), "x": $$c.getField("x")}] [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] + -- ASSIGN |PARTITIONED| + assign [$$49] <- [get-item($$33, 1)] [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] + -- ASSIGN |PARTITIONED| + assign [$$33] <- [$$c.getField("f1")] [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] + -- ASSIGN |PARTITIONED| + project ([$$c]) [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] + -- STREAM_PROJECT |PARTITIONED| + exchange [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + data-scan []<-[$$34, $$c] <- test.ColumnDataset project ({x:any,f1:[{f2:[[any]],f3:[any]}]}) [cardinality: 2.0, op-cost: 2.0, total-cost: 2.0] + -- DATASOURCE_SCAN |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/column/pushdown/other-pushdowns/other-pushdowns.026.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/column/pushdown/other-pushdowns/other-pushdowns.026.plan new file mode 100644 index 0000000000..2379a5ad36 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/column/pushdown/other-pushdowns/other-pushdowns.026.plan @@ -0,0 +1,38 @@ +distribute result [$$32] [cardinality: 2.0, op-cost: 0.0, total-cost: 4.0] +-- DISTRIBUTE_RESULT |PARTITIONED| + exchange [cardinality: 2.0, op-cost: 0.0, total-cost: 4.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + project ([$$32]) [cardinality: 2.0, op-cost: 0.0, total-cost: 4.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$32] <- [{"$1": get-item(get-item($$37, 0), 0), "$2": get-item($$38, 1), "$3": get-item(get-item($$39, 1), 1), "x": $$40}] [cardinality: 2.0, op-cost: 0.0, total-cost: 4.0] + -- ASSIGN |PARTITIONED| + project ([$$37, $$38, $$39, $$40]) [cardinality: 2.0, op-cost: 0.0, total-cost: 4.0] + -- STREAM_PROJECT |PARTITIONED| + exchange [cardinality: 2.0, op-cost: 0.0, total-cost: 4.0] + -- SORT_MERGE_EXCHANGE [$$36(ASC) ] |PARTITIONED| + order (ASC, $$36) [cardinality: 2.0, op-cost: 2.0, total-cost: 4.0] + -- STABLE_SORT [$$36(ASC)] |PARTITIONED| + exchange [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + project ([$$37, $$38, $$39, $$40, $$36]) [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$39, $$38] <- [$$51.getField("f2"), $$51.getField("f3")] [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] + -- ASSIGN |PARTITIONED| + project ([$$36, $$40, $$51, $$37]) [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$51, $$37] <- [get-item($$34, 1), get-item($$34, 0).getField("f2")] [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] + -- ASSIGN |PARTITIONED| + project ([$$34, $$36, $$40]) [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$34, $$36, $$40] <- [$$c.getField("f1"), $$c.getField("y"), $$c.getField("x")] [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] + -- ASSIGN |PARTITIONED| + project ([$$c]) [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] + -- STREAM_PROJECT |PARTITIONED| + exchange [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + data-scan []<-[$$35, $$c] <- test.ColumnDataset project ({x:any,y:any,f1:[{f2:[[any]],f3:[any]}]}) [cardinality: 2.0, op-cost: 2.0, total-cost: 2.0] + -- DATASOURCE_SCAN |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/column/pushdown/other-pushdowns/other-pushdowns.027.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/column/pushdown/other-pushdowns/other-pushdowns.027.plan new file mode 100644 index 0000000000..66b024a654 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/column/pushdown/other-pushdowns/other-pushdowns.027.plan @@ -0,0 +1,26 @@ +distribute result [$$35] [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] +-- DISTRIBUTE_RESULT |PARTITIONED| + exchange [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + project ([$$35]) [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$35] <- [{"$1": get-item(get-item(get-item($$36, 0).getField("f2"), 0), 0), "$2": get-item($$43, 1), "$3": get-item(get-item($$55.getField("f2"), 1), 1), "f4": $$43.getField("f4")}] [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] + -- ASSIGN |PARTITIONED| + assign [$$43] <- [$$55.getField("f3")] [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] + -- ASSIGN |PARTITIONED| + assign [$$55] <- [get-item($$36, 1)] [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] + -- ASSIGN |PARTITIONED| + project ([$$36]) [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$36] <- [$$c.getField("f1")] [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] + -- ASSIGN |PARTITIONED| + project ([$$c]) [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] + -- STREAM_PROJECT |PARTITIONED| + exchange [cardinality: 2.0, op-cost: 0.0, total-cost: 2.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + data-scan []<-[$$37, $$c] <- test.ColumnDataset project ({f1:[{f2:[[any]],f3:<[any],{f4:any}>}]}) [cardinality: 2.0, op-cost: 2.0, total-cost: 2.0] + -- DATASOURCE_SCAN |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/PushdownUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/PushdownUtil.java index b0cc50da5c..c94a86cba5 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/PushdownUtil.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/PushdownUtil.java @@ -88,6 +88,27 @@ public class PushdownUtil { } } + public static int getFieldNameId(AbstractFunctionCallExpression fieldAccessExpr) { + if (!BuiltinFunctions.FIELD_ACCESS_BY_INDEX.equals(fieldAccessExpr.getFunctionIdentifier())) { + return -1; + } + + Integer fieldNameId = ConstantExpressionUtil.getIntArgument(fieldAccessExpr, 1); + if (fieldNameId == null) { + return -1; + } + + return fieldNameId; + } + + public static String getFieldName(AbstractFunctionCallExpression fieldAccessExpr) { + if (!BuiltinFunctions.FIELD_ACCESS_BY_NAME.equals(fieldAccessExpr.getFunctionIdentifier())) { + throw new IllegalStateException( + "Cannot get field name from " + fieldAccessExpr.getFunctionIdentifier().getName()); + } + return ConstantExpressionUtil.getStringArgument(fieldAccessExpr, 1); + } + public static String getFieldName(AbstractFunctionCallExpression fieldAccessExpr, IVariableTypeEnvironment typeEnv) throws AlgebricksException { if (BuiltinFunctions.FIELD_ACCESS_BY_NAME.equals(fieldAccessExpr.getFunctionIdentifier())) {
