Jianfeng Jia has submitted this change and it was merged. Change subject: Intersect the secondary indexes before primary search ......................................................................
Intersect the secondary indexes before primary search Change-Id: Ie167918fb23e39c8728840e4a90c1b85bf1bde85 Reviewed-on: https://asterix-gerrit.ics.uci.edu/578 Tested-by: Jenkins <[email protected]> Reviewed-by: Jianfeng Jia <[email protected]> --- M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IAccessMethod.java M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceSelectAccessMethodRule.java M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanSpecialFlatteningCheckVisitor.java M asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java M asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java A asterix-app/src/test/resources/optimizerts/queries/multi-indexes/btree-rtree-ngram-intersect.aql A asterix-app/src/test/resources/optimizerts/queries/multi-indexes/two-2nd-btree-intersect.aql A asterix-app/src/test/resources/optimizerts/queries/multi-indexes/two-inverted-index-intersect.aql A asterix-app/src/test/resources/optimizerts/queries/multi-indexes/two-rtree-intersect.aql A asterix-app/src/test/resources/optimizerts/queries/multi-indexes/with-primary-index-intersect.aql A asterix-app/src/test/resources/optimizerts/results/multi-indexes/btree-rtree-ngram-intersect.plan A asterix-app/src/test/resources/optimizerts/results/multi-indexes/two-2nd-btree-intersect.plan A asterix-app/src/test/resources/optimizerts/results/multi-indexes/two-inverted-index-intersect.plan A asterix-app/src/test/resources/optimizerts/results/multi-indexes/two-rtree-intersect.plan A asterix-app/src/test/resources/optimizerts/results/multi-indexes/with-primary-index-intersect.plan A asterix-app/src/test/resources/runtimets/queries/index-selection/intersection/tinysocial-intersect.1.ddl.aql A asterix-app/src/test/resources/runtimets/queries/index-selection/intersection/tinysocial-intersect.2.update.aql A asterix-app/src/test/resources/runtimets/queries/index-selection/intersection/tinysocial-intersect.3.query.aql A asterix-app/src/test/resources/runtimets/results/index-selection/intersection/intersection.1.adm M asterix-app/src/test/resources/runtimets/testsuite.xml 28 files changed, 735 insertions(+), 62 deletions(-) Approvals: Jianfeng Jia: Looks good to me, approved Jenkins: Verified diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java index 9a82309..238eaa3 100644 --- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java +++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java @@ -42,6 +42,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteUpsertOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator; @@ -234,6 +235,11 @@ } @Override + public Void visitIntersectOperator(IntersectOperator op, Void arg) throws AlgebricksException { + return null; + } + + @Override public Void visitUnnestOperator(UnnestOperator op, Void arg) throws AlgebricksException { return null; } diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java index 7a36db8..1531b8a 100644 --- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java +++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java @@ -141,14 +141,21 @@ * Simply picks the first index that it finds. TODO: Improve this decision * process by making it more systematic. */ - protected Pair<IAccessMethod, Index> chooseIndex(Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs) { + protected Pair<IAccessMethod, Index> chooseBestIndex(Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs) { + List<Pair<IAccessMethod, Index>> list = chooseAllIndex(analyzedAMs); + return list.isEmpty() ? null : list.get(0); + } + + protected List<Pair<IAccessMethod, Index>> chooseAllIndex( + Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs) { + List<Pair<IAccessMethod, Index>> result = new ArrayList<Pair<IAccessMethod, Index>>(); Iterator<Map.Entry<IAccessMethod, AccessMethodAnalysisContext>> amIt = analyzedAMs.entrySet().iterator(); while (amIt.hasNext()) { Map.Entry<IAccessMethod, AccessMethodAnalysisContext> amEntry = amIt.next(); AccessMethodAnalysisContext analysisCtx = amEntry.getValue(); Iterator<Map.Entry<Index, List<Pair<Integer, Integer>>>> indexIt = analysisCtx.indexExprsAndVars.entrySet() .iterator(); - if (indexIt.hasNext()) { + while (indexIt.hasNext()) { Map.Entry<Index, List<Pair<Integer, Integer>>> indexEntry = indexIt.next(); // To avoid a case where the chosen access method and a chosen // index type is different. @@ -161,24 +168,24 @@ // LENGTH_PARTITIONED_NGRAM_INVIX] IAccessMethod chosenAccessMethod = amEntry.getKey(); Index chosenIndex = indexEntry.getKey(); - boolean isKeywordOrNgramIndexChosen = false; - if (chosenIndex.getIndexType() == IndexType.LENGTH_PARTITIONED_WORD_INVIX + boolean isKeywordOrNgramIndexChosen = + chosenIndex.getIndexType() == IndexType.LENGTH_PARTITIONED_WORD_INVIX || chosenIndex.getIndexType() == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX || chosenIndex.getIndexType() == IndexType.SINGLE_PARTITION_WORD_INVIX - || chosenIndex.getIndexType() == IndexType.SINGLE_PARTITION_NGRAM_INVIX) - isKeywordOrNgramIndexChosen = true; - if ((chosenAccessMethod == BTreeAccessMethod.INSTANCE && chosenIndex.getIndexType() != IndexType.BTREE) + || chosenIndex.getIndexType() == IndexType.SINGLE_PARTITION_NGRAM_INVIX; + + if ((chosenAccessMethod == BTreeAccessMethod.INSTANCE && chosenIndex.getIndexType() == IndexType.BTREE) || (chosenAccessMethod == RTreeAccessMethod.INSTANCE - && chosenIndex.getIndexType() != IndexType.RTREE) - || (chosenAccessMethod == InvertedIndexAccessMethod.INSTANCE && !isKeywordOrNgramIndexChosen)) { - continue; + && chosenIndex.getIndexType() == IndexType.RTREE) + || (chosenAccessMethod == InvertedIndexAccessMethod.INSTANCE && isKeywordOrNgramIndexChosen)) { + result.add(new Pair<IAccessMethod, Index>(chosenAccessMethod, chosenIndex)); } - return new Pair<IAccessMethod, Index>(chosenAccessMethod, chosenIndex); } } - return null; + return result; } + /** * Removes irrelevant access methods candidates, based on whether the * expressions in the query match those in the index. For example, some diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java index dd6784e..370b90d 100644 --- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java +++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java @@ -59,9 +59,11 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator; +import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil; /** * Class for helping rewrite rules to choose and apply BTree indexes. @@ -123,7 +125,7 @@ IOptimizationContext context) throws AlgebricksException { SelectOperator select = (SelectOperator) selectRef.getValue(); Mutable<ILogicalExpression> conditionRef = select.getCondition(); - ILogicalOperator primaryIndexUnnestOp = createSecondaryToPrimaryPlan(selectRef, conditionRef, subTree, null, + ILogicalOperator primaryIndexUnnestOp = createSecondaryToPrimaryPlan(conditionRef, subTree, null, chosenIndex, analysisCtx, false, false, false, context); if (primaryIndexUnnestOp == null) { return false; @@ -184,7 +186,7 @@ newNullPlaceHolderVar = indexSubTree.getDataSourceVariables().get(0); } - ILogicalOperator primaryIndexUnnestOp = createSecondaryToPrimaryPlan(joinRef, conditionRef, indexSubTree, + ILogicalOperator primaryIndexUnnestOp = createSecondaryToPrimaryPlan(conditionRef, indexSubTree, probeSubTree, chosenIndex, analysisCtx, true, isLeftOuterJoin, true, context); if (primaryIndexUnnestOp == null) { return false; @@ -210,8 +212,8 @@ return true; } - private ILogicalOperator createSecondaryToPrimaryPlan(Mutable<ILogicalOperator> topOpRef, - Mutable<ILogicalExpression> conditionRef, OptimizableOperatorSubTree indexSubTree, + @Override + public ILogicalOperator createSecondaryToPrimaryPlan(Mutable<ILogicalExpression> conditionRef, OptimizableOperatorSubTree indexSubTree, OptimizableOperatorSubTree probeSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx, boolean retainInput, boolean retainNull, boolean requiresBroadcast, IOptimizationContext context) throws AlgebricksException { @@ -470,7 +472,8 @@ // Assign operator that sets the constant secondary-index search-key fields if necessary. AssignOperator assignConstantSearchKeys = new AssignOperator(assignKeyVarList, assignKeyExprList); // Input to this assign is the EmptyTupleSource (which the dataSourceScan also must have had as input). - assignConstantSearchKeys.getInputs().add(dataSourceOp.getInputs().get(0)); + assignConstantSearchKeys.getInputs().add(new MutableObject<ILogicalOperator>( + OperatorManipulationUtil.deepCopyWithExcutionMode(dataSourceOp.getInputs().get(0).getValue()))); assignConstantSearchKeys.setExecutionMode(dataSourceOp.getExecutionMode()); inputOp = assignConstantSearchKeys; } else { @@ -489,15 +492,10 @@ ExternalDataLookupOperator externalDataAccessOp = AccessMethodUtils.createExternalDataLookupUnnestMap( dataSourceOp, dataset, recordType, secondaryIndexUnnestOp, context, chosenIndex, retainInput, retainNull); - indexSubTree.dataSourceRef.setValue(externalDataAccessOp); return externalDataAccessOp; } else if (!isPrimaryIndex) { primaryIndexUnnestOp = AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceOp, dataset, recordType, secondaryIndexUnnestOp, context, true, retainInput, retainNull, false); - - // Replace the datasource scan with the new plan rooted at - // primaryIndexUnnestMap. - indexSubTree.dataSourceRef.setValue(primaryIndexUnnestOp); // Adds equivalence classes --- one equivalent class between a primary key // variable and a record field-access expression. @@ -678,4 +676,15 @@ // No additional analysis required for BTrees. return true; } + + @Override + public String getName() { + return "BTREE_ACCESS_METHOD"; + } + + @Override + public int compareTo(IAccessMethod o) { + return this.getName().compareTo(o.getName()); + } + } diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IAccessMethod.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IAccessMethod.java index 6538ead..5691d57 100644 --- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IAccessMethod.java +++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IAccessMethod.java @@ -23,12 +23,14 @@ import org.apache.asterix.metadata.entities.Index; import org.apache.commons.lang3.mutable.Mutable; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; +import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator; import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext; import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression; import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment; import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator; +import org.apache.hyracks.data.std.api.IDataOutputProvider; /** * Interface that an access method should implement to work with the rewrite @@ -36,7 +38,7 @@ * methods for analyzing a select/join condition, and for rewriting the plan * with a given index. */ -public interface IAccessMethod { +public interface IAccessMethod extends Comparable<IAccessMethod>{ /** * @return A list of function identifiers that are optimizable by this @@ -83,6 +85,15 @@ OptimizableOperatorSubTree subTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx, IOptimizationContext context) throws AlgebricksException; + public ILogicalOperator createSecondaryToPrimaryPlan(Mutable<ILogicalExpression> conditionRef, + OptimizableOperatorSubTree indexSubTree, + OptimizableOperatorSubTree probeSubTree, + Index chosenIndex, + AccessMethodAnalysisContext analysisCtx, + boolean retainInput, boolean retainNull, boolean requiresBroadcast, + IOptimizationContext context) + throws AlgebricksException; + /** * Applies the plan transformation to use chosenIndex to optimize a join query. * In the case of a LeftOuterJoin, there may or may not be a needed groupby operation @@ -100,4 +111,6 @@ */ public boolean exprIsOptimizable(Index index, IOptimizableFuncExpr optFuncExpr) throws AlgebricksException; + public String getName(); + } diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java index 13f7119..fce84d1 100644 --- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java +++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java @@ -145,7 +145,7 @@ removeIndexCandidatesFromOuterBranch(analyzedAMs); // Choose an index from the inner branch that will be used. - Pair<IAccessMethod, Index> chosenIndex = chooseIndex(analyzedAMs); + Pair<IAccessMethod, Index> chosenIndex = chooseBestIndex(analyzedAMs); if (chosenIndex == null) { context.addToDontApplySet(this, join); return false; diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceSelectAccessMethodRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceSelectAccessMethodRule.java index 40e8712..405e4aa 100644 --- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceSelectAccessMethodRule.java +++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceSelectAccessMethodRule.java @@ -18,13 +18,17 @@ */ package org.apache.asterix.optimizer.rules.am; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; import org.apache.asterix.metadata.declared.AqlMetadataProvider; import org.apache.asterix.metadata.entities.Index; import org.apache.commons.lang3.mutable.Mutable; +import org.apache.commons.lang3.mutable.MutableObject; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.common.utils.Pair; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; @@ -32,10 +36,14 @@ import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext; import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag; import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag; +import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable; import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression; import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment; +import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression; import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator; import org.apache.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil; @@ -62,6 +70,9 @@ * 3. Check metadata to see if there are applicable indexes. * 4. Choose an index to apply (for now only a single index will be chosen). * 5. Rewrite plan using index (delegated to IAccessMethods). + * If there are multiple secondary index access path available, we will use the intersection operator to get the + * intersected primary key from all the secondary indexes. The detailed documentation is here + * https://cwiki.apache.org/confluence/display/ASTERIXDB/Intersect+multiple+secondary+index */ public class IntroduceSelectAccessMethodRule extends AbstractIntroduceAccessMethodRule { @@ -94,7 +105,7 @@ } // Analyze select condition. - Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs = new HashMap<IAccessMethod, AccessMethodAnalysisContext>(); + Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs = new TreeMap<IAccessMethod, AccessMethodAnalysisContext>(); if (!analyzeCondition(selectCond, subTree.assignsAndUnnests, analyzedAMs, context, typeEnvironment)) { return false; } @@ -108,16 +119,15 @@ pruneIndexCandidates(analyzedAMs, context, typeEnvironment); // Choose index to be applied. - Pair<IAccessMethod, Index> chosenIndex = chooseIndex(analyzedAMs); - if (chosenIndex == null) { + List<Pair<IAccessMethod, Index>> chosenIndexes = chooseAllIndex(analyzedAMs); + if (chosenIndexes == null || chosenIndexes.size() == 0) { context.addToDontApplySet(this, select); return false; } // Apply plan transformation using chosen index. - AccessMethodAnalysisContext analysisCtx = analyzedAMs.get(chosenIndex.first); - boolean res = chosenIndex.first.applySelectPlanTransformation(selectRef, subTree, chosenIndex.second, - analysisCtx, context); + boolean res = intersectAllSecondaryIndexes(chosenIndexes, analyzedAMs, context); + if (res) { OperatorPropertiesUtil.typeOpRec(opRef, context); } @@ -125,6 +135,76 @@ return res; } + private boolean intersectAllSecondaryIndexes(List<Pair<IAccessMethod, Index>> chosenIndexes, + Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs, IOptimizationContext context) + throws AlgebricksException { + Pair<IAccessMethod, Index> chosenIndex = null; + Optional<Pair<IAccessMethod, Index>> primaryIndex = chosenIndexes.stream() + .filter(pair -> pair.second.isPrimaryIndex()).findFirst(); + if (chosenIndexes.size() == 1) { + chosenIndex = chosenIndexes.get(0); + } else if (primaryIndex.isPresent()) { + // one primary + secondary indexes, choose the primary index directly. + chosenIndex = primaryIndex.get(); + } + if (chosenIndex != null) { + AccessMethodAnalysisContext analysisCtx = analyzedAMs.get(chosenIndex.first); + return chosenIndex.first.applySelectPlanTransformation(selectRef, subTree, chosenIndex.second, analysisCtx, + context); + } + + // Intersect all secondary indexes, and postpone the primary index search. + Mutable<ILogicalExpression> conditionRef = select.getCondition(); + + List<ILogicalOperator> subRoots = new ArrayList<>(); + for (Pair<IAccessMethod, Index> pair : chosenIndexes) { + AccessMethodAnalysisContext analysisCtx = analyzedAMs.get(pair.first); + subRoots.add(pair.first.createSecondaryToPrimaryPlan(conditionRef, subTree, null, pair.second, analysisCtx, + true, true, false, context)); + } + ILogicalOperator primaryUnnest = connectAll2ndarySearchPlanWithIntersect(subRoots, context); + + subTree.dataSourceRef.setValue(primaryUnnest); + return primaryUnnest != null; + } + + private ILogicalOperator connectAll2ndarySearchPlanWithIntersect(List<ILogicalOperator> subRoots, + IOptimizationContext context) throws AlgebricksException { + ILogicalOperator lop = subRoots.get(0); + List<List<LogicalVariable>> inputVars = new ArrayList<>(subRoots.size()); + for (int i = 0; i < subRoots.size(); i++) { + if (lop.getOperatorTag() != subRoots.get(i).getOperatorTag()) { + throw new AlgebricksException("The data source root should have the same operator type"); + } + if (lop.getInputs().size() != 1) { + throw new AlgebricksException("The primary search has multiple input"); + } + + ILogicalOperator curRoot = subRoots.get(i); + OrderOperator order = (OrderOperator) curRoot.getInputs().get(0).getValue(); + List<LogicalVariable> orderedColumn = new ArrayList<>(order.getOrderExpressions().size()); + for (Pair<OrderOperator.IOrder, Mutable<ILogicalExpression>> orderExpression : order + .getOrderExpressions()) { + if (orderExpression.second.getValue().getExpressionTag() != LogicalExpressionTag.VARIABLE) { + throw new AlgebricksException("It should not happen, the order by expression is not variables"); + } + VariableReferenceExpression orderedVar = (VariableReferenceExpression) orderExpression.second + .getValue(); + orderedColumn.add(orderedVar.getVariableReference()); + } + inputVars.add(orderedColumn); + } + + List<LogicalVariable> outputVar = inputVars.get(0); + IntersectOperator intersect = new IntersectOperator(outputVar, inputVars); + for (ILogicalOperator secondarySearch : subRoots) { + intersect.getInputs().add(secondarySearch.getInputs().get(0)); + } + context.computeAndSetTypeEnvironmentForOperator(intersect); + lop.getInputs().set(0, new MutableObject<>(intersect)); + return lop; + } + protected boolean matchesOperatorPattern(Mutable<ILogicalOperator> opRef, IOptimizationContext context) { // First check that the operator is a select and its condition is a function call. AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue(); diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java index d587649..d8e4c6a 100644 --- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java +++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java @@ -66,6 +66,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator; @@ -74,6 +75,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.LogicalOperatorDeepCopyWithNewVariablesVisitor; import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities; +import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexSearchModifierFactory; import org.apache.hyracks.storage.am.lsm.invertedindex.search.ConjunctiveEditDistanceSearchModifierFactory; import org.apache.hyracks.storage.am.lsm.invertedindex.search.ConjunctiveListEditDistanceSearchModifierFactory; @@ -359,10 +361,14 @@ return false; } - private ILogicalOperator createSecondaryToPrimaryPlan(OptimizableOperatorSubTree indexSubTree, - OptimizableOperatorSubTree probeSubTree, Index chosenIndex, IOptimizableFuncExpr optFuncExpr, + @Override + public ILogicalOperator createSecondaryToPrimaryPlan(Mutable<ILogicalExpression> conditionRef, + OptimizableOperatorSubTree indexSubTree, OptimizableOperatorSubTree probeSubTree, Index chosenIndex, + AccessMethodAnalysisContext analysisCtx, boolean retainInput, boolean retainNull, boolean requiresBroadcast, IOptimizationContext context) throws AlgebricksException { + + IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx); Dataset dataset = indexSubTree.dataset; ARecordType recordType = indexSubTree.recordType; // we made sure indexSubTree has datasource scan @@ -390,7 +396,8 @@ // Assign operator that sets the secondary-index search-key fields. inputOp = new AssignOperator(keyVarList, keyExprList); // Input to this assign is the EmptyTupleSource (which the dataSourceScan also must have had as input). - inputOp.getInputs().add(dataSourceScan.getInputs().get(0)); + inputOp.getInputs().add(new MutableObject<>( + OperatorManipulationUtil.deepCopyWithExcutionMode(dataSourceScan.getInputs().get(0).getValue()))); inputOp.setExecutionMode(dataSourceScan.getExecutionMode()); } else { // We are optimizing a join. Add the input variable to the secondaryIndexFuncArgs. @@ -429,8 +436,7 @@ public boolean applySelectPlanTransformation(Mutable<ILogicalOperator> selectRef, OptimizableOperatorSubTree subTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx, IOptimizationContext context) throws AlgebricksException { - IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx); - ILogicalOperator indexPlanRootOp = createSecondaryToPrimaryPlan(subTree, null, chosenIndex, optFuncExpr, false, + ILogicalOperator indexPlanRootOp = createSecondaryToPrimaryPlan(null, subTree, null, chosenIndex, analysisCtx, false, false, false, context); // Replace the datasource scan with the new plan rooted at primaryIndexUnnestMap. subTree.dataSourceRef.setValue(indexPlanRootOp); @@ -511,8 +517,8 @@ probeSubTree.root = newProbeRootRef.getValue(); } // Create regular indexed-nested loop join path. - ILogicalOperator indexPlanRootOp = createSecondaryToPrimaryPlan(indexSubTree, probeSubTree, chosenIndex, - optFuncExpr, true, isLeftOuterJoin, true, context); + ILogicalOperator indexPlanRootOp = createSecondaryToPrimaryPlan(null, indexSubTree, probeSubTree, chosenIndex, + analysisCtx, true, isLeftOuterJoin, true, context); indexSubTree.dataSourceRef.setValue(indexPlanRootOp); // Change join into a select with the same condition. @@ -1181,4 +1187,14 @@ } context.computeAndSetTypeEnvironmentForOperator(op); } + + @Override + public String getName() { + return "INVERTED_INDEX_ACCESS_METHOD"; + } + + @Override + public int compareTo(IAccessMethod o) { + return this.getName().compareTo(o.getName()); + } } diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java index 461fb2a..c5bd8af 100644 --- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java +++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java @@ -54,6 +54,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator; +import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil; /** * Class for helping rewrite rules to choose and apply RTree indexes. @@ -100,15 +101,23 @@ OptimizableOperatorSubTree subTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx, IOptimizationContext context) throws AlgebricksException { // TODO: We can probably do something smarter here based on selectivity or MBR area. - IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx); - ILogicalOperator primaryIndexUnnestOp = createSecondaryToPrimaryPlan(subTree, null, chosenIndex, optFuncExpr, - analysisCtx, false, false, false, context); + ILogicalOperator primaryIndexUnnestOp = createSecondaryToPrimaryPlan(subTree, null, chosenIndex, analysisCtx, + false, false, false, context); if (primaryIndexUnnestOp == null) { return false; } // Replace the datasource scan with the new plan rooted at primaryIndexUnnestMap. subTree.dataSourceRef.setValue(primaryIndexUnnestOp); return true; + } + + @Override + public ILogicalOperator createSecondaryToPrimaryPlan(Mutable<ILogicalExpression> conditionRef, + OptimizableOperatorSubTree indexSubTree, OptimizableOperatorSubTree probeSubTree, Index chosenIndex, + AccessMethodAnalysisContext analysisCtx, boolean retainInput, boolean retainNull, boolean requiresBroadcast, + IOptimizationContext context) throws AlgebricksException { + return createSecondaryToPrimaryPlan(indexSubTree, probeSubTree, chosenIndex, analysisCtx, retainInput, + retainNull, requiresBroadcast, context); } @Override @@ -140,9 +149,8 @@ } // TODO: We can probably do something smarter here based on selectivity or MBR area. - IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx); ILogicalOperator primaryIndexUnnestOp = createSecondaryToPrimaryPlan(indexSubTree, probeSubTree, chosenIndex, - optFuncExpr, analysisCtx, true, isLeftOuterJoin, true, context); + analysisCtx, true, isLeftOuterJoin, true, context); if (primaryIndexUnnestOp == null) { return false; } @@ -165,9 +173,11 @@ } private ILogicalOperator createSecondaryToPrimaryPlan(OptimizableOperatorSubTree indexSubTree, - OptimizableOperatorSubTree probeSubTree, Index chosenIndex, IOptimizableFuncExpr optFuncExpr, - AccessMethodAnalysisContext analysisCtx, boolean retainInput, boolean retainNull, boolean requiresBroadcast, - IOptimizationContext context) throws AlgebricksException { + OptimizableOperatorSubTree probeSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx, + boolean retainInput, boolean retainNull, boolean requiresBroadcast, IOptimizationContext context) + throws AlgebricksException { + + IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx); Dataset dataset = indexSubTree.dataset; ARecordType recordType = indexSubTree.recordType; @@ -221,7 +231,8 @@ if (probeSubTree == null) { // We are optimizing a selection query. // Input to this assign is the EmptyTupleSource (which the dataSourceScan also must have had as input). - assignSearchKeys.getInputs().add(dataSourceOp.getInputs().get(0)); + assignSearchKeys.getInputs().add(new MutableObject<ILogicalOperator>( + OperatorManipulationUtil.deepCopyWithExcutionMode(dataSourceOp.getInputs().get(0).getValue()))); assignSearchKeys.setExecutionMode(dataSourceOp.getExecutionMode()); } else { // We are optimizing a join, place the assign op top of the probe subtree. @@ -254,4 +265,15 @@ // No additional analysis required. return true; } + + @Override + public String getName() { + return "RTREE_ACCESS_METHOD"; + } + + @Override + public int compareTo(IAccessMethod o) { + return this.getName().compareTo(o.getName()); + } + } diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java index 42905ee..0715d46 100644 --- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java +++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java @@ -57,6 +57,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator; @@ -528,6 +529,22 @@ } @Override + public ILogicalOperator visitIntersectOperator(IntersectOperator op, Void arg) throws AlgebricksException { + visitMultiInputOperator(op); + List<LogicalVariable> outputVars = op.getOutputVars(); + for (int i = 0; i < op.getNumInput(); i++) { + List<LogicalVariable> inputVars = op.getInputVariables(i); + if (inputVars.size() != outputVars.size()) { + throw new AlgebricksException("The cardinality of input and output are not equal for Intersection"); + } + for (int j = 0; j < inputVars.size(); j++) { + updateInputToOutputVarMapping(inputVars.get(j), outputVars.get(j), false); + } + } + return op; + } + + @Override public ILogicalOperator visitUnnestOperator(UnnestOperator op, Void arg) throws AlgebricksException { return visitSingleInputOperator(op); } diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java index 191a2f9..5aaa59f 100644 --- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java +++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java @@ -45,6 +45,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator; @@ -307,6 +308,12 @@ } @Override + public ILogicalOperator visitIntersectOperator(IntersectOperator op, Void arg) throws AlgebricksException { + throw new UnsupportedOperationException( + "Nested subplans with a intersect operator should have been disqualified for this rewriting!"); + } + + @Override public ILogicalOperator visitUnnestOperator(UnnestOperator op, Void arg) throws AlgebricksException { return visitSingleInputOperator(op); } diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanSpecialFlatteningCheckVisitor.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanSpecialFlatteningCheckVisitor.java index 1a2e8bb..c691365 100644 --- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanSpecialFlatteningCheckVisitor.java +++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanSpecialFlatteningCheckVisitor.java @@ -31,6 +31,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator; @@ -184,6 +185,11 @@ } @Override + public Boolean visitIntersectOperator(IntersectOperator op, Void arg) throws AlgebricksException { + return false; + } + + @Override public Boolean visitUnnestOperator(UnnestOperator op, Void arg) throws AlgebricksException { return visitInputs(op); } diff --git a/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java b/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java index 538e7ca..c62f5d9 100644 --- a/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java +++ b/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java @@ -95,26 +95,34 @@ AsterixHyracksIntegrationUtil.deinit(true); } - private static void suiteBuild(File dir, Collection<Object[]> testArgs, String path) { - for (File file : dir.listFiles()) { - if (file.isDirectory() && !file.getName().startsWith(".")) { - suiteBuild(file, testArgs, path + file.getName() + SEPARATOR); + private static void suiteBuildPerFile(File file, Collection<Object[]> testArgs, String path) { + if (file.isDirectory() && !file.getName().startsWith(".")) { + for (File innerfile : file.listFiles()) { + String subdir = innerfile.isDirectory() ? path + innerfile.getName() + SEPARATOR : path; + suiteBuildPerFile(innerfile, testArgs, subdir); } - if (file.isFile() && file.getName().endsWith(EXTENSION_QUERY) - // && !ignore.contains(path + file.getName()) - ) { - String resultFileName = AsterixTestHelper.extToResExt(file.getName(), EXTENSION_RESULT); - File expectedFile = new File(PATH_EXPECTED + path + resultFileName); - File actualFile = new File(PATH_ACTUAL + SEPARATOR + path.replace(SEPARATOR, "_") + resultFileName); - testArgs.add(new Object[] { file, expectedFile, actualFile }); - } + } + if (file.isFile() && file.getName().endsWith(EXTENSION_QUERY) + // && !ignore.contains(path + file.getName()) + ) { + String resultFileName = AsterixTestHelper.extToResExt(file.getName(), EXTENSION_RESULT); + File expectedFile = new File(PATH_EXPECTED + path + resultFileName); + File actualFile = new File(PATH_ACTUAL + SEPARATOR + path.replace(SEPARATOR, "_") + resultFileName); + testArgs.add(new Object[] { file, expectedFile, actualFile }); } } @Parameters(name = "OptimizerTest {index}: {0}") public static Collection<Object[]> tests() { Collection<Object[]> testArgs = new ArrayList<Object[]>(); - suiteBuild(new File(PATH_QUERIES), testArgs, ""); + if (only.isEmpty()) { + suiteBuildPerFile(new File(PATH_QUERIES), testArgs, ""); + } else { + for (String path : only) { + suiteBuildPerFile(new File(PATH_QUERIES + path), testArgs, + path.lastIndexOf(SEPARATOR) < 0 ? "" : path.substring(0, path.lastIndexOf(SEPARATOR) + 1)); + } + } return testArgs; } diff --git a/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java b/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java index 4c79965..21fc240 100644 --- a/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java +++ b/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java @@ -52,8 +52,6 @@ protected static AsterixTransactionProperties txnProperties; private final static TestExecutor testExecutor = new TestExecutor(); - protected static TestGroup FailedGroup; - @BeforeClass public static void setUp() throws Exception { try { @@ -98,6 +96,6 @@ @Test public void test() throws Exception { - testExecutor.executeTest(PATH_ACTUAL, tcCtx, null, false, FailedGroup); + testExecutor.executeTest(PATH_ACTUAL, tcCtx, null, false, ExecutionTestUtil.FailedGroup); } } diff --git a/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/btree-rtree-ngram-intersect.aql b/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/btree-rtree-ngram-intersect.aql new file mode 100644 index 0000000..5f57d07 --- /dev/null +++ b/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/btree-rtree-ngram-intersect.aql @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Tests three types of secondary indexes should trigger intersection rule + * Success : Yes + */ + +drop dataverse test if exists; +create dataverse test; +use dataverse test; + +create type tTweet as closed { + id: int32, + location: point, + message: string, + create_at: datetime, + misc: string +} + +create dataset dsTweet(tTweet) primary key id; + +create index ngram_index on dsTweet(message) type ngram(3); +create index time_index on dsTweet(create_at) type btree; +create index location_index on dsTweet(location) type rtree; + +write output to nc1:"rttest/btree-rtree-ngram-intersect.adm"; + +let $region := create-rectangle(create-point(-128.43007812500002,20.298506037222175), create-point(-64.26992187500002,54.56902589732035)) +let $ts_start := datetime("2015-11-11T00:00:00Z") +let $ts_end := datetime("2015-12-18T23:59:59Z") +let $keyword := "hello" +for $t in dataset dsTweet +where $t.create_at >= $ts_start and $t.create_at < $ts_end + and spatial-intersect($t.location, $region) + and contains($t.message, $keyword) +return $t + diff --git a/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/two-2nd-btree-intersect.aql b/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/two-2nd-btree-intersect.aql new file mode 100644 index 0000000..f0ecf57 --- /dev/null +++ b/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/two-2nd-btree-intersect.aql @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Tests two secondary Btree indexes should trigger intersection rule + * Success : Yes + */ + +drop dataverse test if exists; +create dataverse test; +use dataverse test; + +create type tTweet as closed { + id: int32, + location: point, + message: string, + create_at: datetime, + misc: string +} + +create dataset dsTweet(tTweet) primary key id; + +create index time_index on dsTweet(create_at) type btree; +create index misc_index on dsTweet(misc) type btree; + +write output to nc1:"rttest/two-2nd-btree-intersect.adm"; + +let $ts_start := datetime("2015-11-11T00:00:00Z") +let $ts_end := datetime("2015-12-18T23:59:59Z") +let $keyword := "hello" +for $t in dataset dsTweet +where $t.create_at >= $ts_start and $t.create_at < $ts_end + and $t.misc > $keyword +return $t + diff --git a/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/two-inverted-index-intersect.aql b/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/two-inverted-index-intersect.aql new file mode 100644 index 0000000..2947d61 --- /dev/null +++ b/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/two-inverted-index-intersect.aql @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Tests keyword index and the ngram indexe should trigger intersection rule + * Success : Yes + */ + +drop dataverse test if exists; +create dataverse test; +use dataverse test; + +create type tTweet as closed { + id: int32, + message: string, + create_at: datetime, + misc: string +} + +create dataset dsTweet(tTweet) primary key id; + +create index msg_index on dsTweet(message) type ngram(3); +create index msc_index on dsTweet(misc) type keyword; + +for $t in dataset dsTweet +let $ed := edit-distance($t.message, "Suzanna Tilson") +let $jacc := similarity-jaccard-check(word-tokens($t.misc), word-tokens("love like verizon"), 0.2f) +where $jacc[0] and $ed <= 2 +return $t + diff --git a/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/two-rtree-intersect.aql b/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/two-rtree-intersect.aql new file mode 100644 index 0000000..40c9617 --- /dev/null +++ b/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/two-rtree-intersect.aql @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Tests two secondary rtree indexes should trigger intersection rule + * Success : Yes + */ + +drop dataverse test if exists; +create dataverse test; +use dataverse test; + +create type tTweet as closed { + id: int32, + location: point, + place: rectangle, + misc: string +} + +create dataset dsTweet(tTweet) primary key id; + +create index loc_index on dsTweet(location) type rtree; +create index plc_index on dsTweet(place) type rtree; + +let $region := create-rectangle(create-point(-128.43007812500002,20.298506037222175), create-point(-64.26992187500002,54.56902589732035)) +for $t in dataset dsTweet +where spatial-intersect($t.location , $region) and spatial-intersect($t.place, $region) +return $t + diff --git a/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/with-primary-index-intersect.aql b/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/with-primary-index-intersect.aql new file mode 100644 index 0000000..d90f119 --- /dev/null +++ b/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/with-primary-index-intersect.aql @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Tests primary index will be chosen if it existed in the job plan, no intersection should apply + * Success : Yes + */ + +drop dataverse test if exists; +create dataverse test; +use dataverse test; + +create type tTweet as closed { + id: int32, + location: point, + message: string, + create_at: datetime, + misc: string +} + +create dataset dsTweet(tTweet) primary key id; + +create index ngram_index on dsTweet(message) type ngram(3); +create index time_index on dsTweet(create_at) type btree; +create index location_index on dsTweet(location) type rtree; + +write output to nc1:"rttest/btree-rtree-ngram-intersect.adm"; + +let $region := create-rectangle(create-point(-128.43007812500002,20.298506037222175), create-point(-64.26992187500002,54.56902589732035)) +let $ts_start := datetime("2015-11-11T00:00:00Z") +let $ts_end := datetime("2015-12-18T23:59:59Z") +let $keyword := "hello" +for $t in dataset dsTweet +where $t.create_at >= $ts_start and $t.create_at < $ts_end + and spatial-intersect($t.location, $region) + and contains($t.message, $keyword) + and $t.id > 0 +return $t + diff --git a/asterix-app/src/test/resources/optimizerts/results/multi-indexes/btree-rtree-ngram-intersect.plan b/asterix-app/src/test/resources/optimizerts/results/multi-indexes/btree-rtree-ngram-intersect.plan new file mode 100644 index 0000000..2d74649 --- /dev/null +++ b/asterix-app/src/test/resources/optimizerts/results/multi-indexes/btree-rtree-ngram-intersect.plan @@ -0,0 +1,37 @@ +-- DISTRIBUTE_RESULT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- STREAM_SELECT |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- BTREE_SEARCH |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- INTERSECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STABLE_SORT [$$29(ASC)] |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- BTREE_SEARCH |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STABLE_SORT [$$31(ASC)] |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- LENGTH_PARTITIONED_INVERTED_INDEX_SEARCH |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STABLE_SORT [$$40(ASC)] |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- RTREE_SEARCH |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --git a/asterix-app/src/test/resources/optimizerts/results/multi-indexes/two-2nd-btree-intersect.plan b/asterix-app/src/test/resources/optimizerts/results/multi-indexes/two-2nd-btree-intersect.plan new file mode 100644 index 0000000..5adb550 --- /dev/null +++ b/asterix-app/src/test/resources/optimizerts/results/multi-indexes/two-2nd-btree-intersect.plan @@ -0,0 +1,28 @@ +-- DISTRIBUTE_RESULT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- STREAM_SELECT |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- BTREE_SEARCH |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- INTERSECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STABLE_SORT [$$19(ASC)] |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- BTREE_SEARCH |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STABLE_SORT [$$23(ASC)] |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- BTREE_SEARCH |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --git a/asterix-app/src/test/resources/optimizerts/results/multi-indexes/two-inverted-index-intersect.plan b/asterix-app/src/test/resources/optimizerts/results/multi-indexes/two-inverted-index-intersect.plan new file mode 100644 index 0000000..7d99635 --- /dev/null +++ b/asterix-app/src/test/resources/optimizerts/results/multi-indexes/two-inverted-index-intersect.plan @@ -0,0 +1,26 @@ +-- DISTRIBUTE_RESULT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_SELECT |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- BTREE_SEARCH |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- INTERSECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STABLE_SORT [$$18(ASC)] |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- LENGTH_PARTITIONED_INVERTED_INDEX_SEARCH |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STABLE_SORT [$$20(ASC)] |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- LENGTH_PARTITIONED_INVERTED_INDEX_SEARCH |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --git a/asterix-app/src/test/resources/optimizerts/results/multi-indexes/two-rtree-intersect.plan b/asterix-app/src/test/resources/optimizerts/results/multi-indexes/two-rtree-intersect.plan new file mode 100644 index 0000000..05d43a9 --- /dev/null +++ b/asterix-app/src/test/resources/optimizerts/results/multi-indexes/two-rtree-intersect.plan @@ -0,0 +1,32 @@ +-- DISTRIBUTE_RESULT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_SELECT |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- BTREE_SEARCH |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- INTERSECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STABLE_SORT [$$24(ASC)] |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- RTREE_SEARCH |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- SPLIT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STABLE_SORT [$$33(ASC)] |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- RTREE_SEARCH |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- SPLIT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --git a/asterix-app/src/test/resources/optimizerts/results/multi-indexes/with-primary-index-intersect.plan b/asterix-app/src/test/resources/optimizerts/results/multi-indexes/with-primary-index-intersect.plan new file mode 100644 index 0000000..7978f8a --- /dev/null +++ b/asterix-app/src/test/resources/optimizerts/results/multi-indexes/with-primary-index-intersect.plan @@ -0,0 +1,11 @@ +-- DISTRIBUTE_RESULT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- STREAM_SELECT |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- BTREE_SEARCH |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --git a/asterix-app/src/test/resources/runtimets/queries/index-selection/intersection/tinysocial-intersect.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/index-selection/intersection/tinysocial-intersect.1.ddl.aql new file mode 100644 index 0000000..10145ed --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/queries/index-selection/intersection/tinysocial-intersect.1.ddl.aql @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +drop dataverse TinySocial if exists; +create dataverse TinySocial; +use dataverse TinySocial; + +create type TwitterUserType as open { + screen-name: string, + lang: string, + friends_count: int64, + statuses_count: int64, + name: string, + followers_count: int64, + sender-location: point +} + +create type TweetMessageType as closed { + tweetid: int64, + user: TwitterUserType, + send-time: datetime, + referred-topics: {{ string }}, + message-text: string +} + +create dataset TweetMessages(TweetMessageType) +primary key tweetid +hints(cardinality=100); + +create index twTimeIdx on TweetMessages(send-time) type btree; +create index twLocationIdx on TweetMessages(user.sender-location) type rtree; +create index twMessage on TweetMessages(message-text) type ngram(2); diff --git a/asterix-app/src/test/resources/runtimets/queries/index-selection/intersection/tinysocial-intersect.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/index-selection/intersection/tinysocial-intersect.2.update.aql new file mode 100644 index 0000000..9923ff4 --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/queries/index-selection/intersection/tinysocial-intersect.2.update.aql @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use dataverse TinySocial; + +load dataset TweetMessages using localfs +(("path"="asterix_nc1://data/tinysocial/twm-nested.adm"),("format"="adm")); diff --git a/asterix-app/src/test/resources/runtimets/queries/index-selection/intersection/tinysocial-intersect.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/index-selection/intersection/tinysocial-intersect.3.query.aql new file mode 100644 index 0000000..392008c --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/queries/index-selection/intersection/tinysocial-intersect.3.query.aql @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use dataverse TinySocial; + +let $ts := datetime("2010-12-12T00:00:00Z") +let $region := create-rectangle(create-point(0.0,0.0),create-point(100.0,100.0)) +let $keyword := "verizon" +for $t in dataset TweetMessages +where $t.send-time > $ts + and spatial-intersect($t.user.sender-location, $region) + and contains($t.message-text, $keyword) +return $t diff --git a/asterix-app/src/test/resources/runtimets/results/index-selection/intersection/intersection.1.adm b/asterix-app/src/test/resources/runtimets/results/index-selection/intersection/intersection.1.adm new file mode 100644 index 0000000..44cc08a --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/results/index-selection/intersection/intersection.1.adm @@ -0,0 +1 @@ +{ "tweetid": 9, "user": { "screen-name": "NathanGiesen@211", "lang": "en", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416, "sender-location": point("36.86,74.62") }, "send-time": datetime("2012-07-21T10:10:00.000Z"), "referred-topics": {{ "verizon", "voicemail-service" }}, "message-text": " love verizon its voicemail-service is awesome" } diff --git a/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterix-app/src/test/resources/runtimets/testsuite.xml index f148a77..99b195d 100644 --- a/asterix-app/src/test/resources/runtimets/testsuite.xml +++ b/asterix-app/src/test/resources/runtimets/testsuite.xml @@ -2602,6 +2602,11 @@ <output-dir compare="Text">disjunctive-predicate-1</output-dir> </compilation-unit> </test-case> + <test-case FilePath="index-selection"> + <compilation-unit name="intersection"> + <output-dir compare="Text">intersection</output-dir> + </compilation-unit> + </test-case> </test-group> <test-group name="inverted-index-join"> <test-case FilePath="inverted-index-join"> -- To view, visit https://asterix-gerrit.ics.uci.edu/578 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ie167918fb23e39c8728840e4a90c1b85bf1bde85 Gerrit-PatchSet: 7 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Jianfeng Jia <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Jianfeng Jia <[email protected]> Gerrit-Reviewer: Taewoo Kim <[email protected]> Gerrit-Reviewer: Yingyi Bu <[email protected]>
