Jianfeng Jia has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/578
Change subject: Intersect the 2ndary indexes before primary search
......................................................................
Intersect the 2ndary indexes before primary search
Change-Id: Ie167918fb23e39c8728840e4a90c1b85bf1bde85
---
M
asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceUnionRule.java
M
asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
M
asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
M
asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
M
asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IAccessMethod.java
M
asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java
M
asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceSelectAccessMethodRule.java
M
asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
M
asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
M asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java
M asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java
A
asterix-app/src/test/resources/optimizerts/queries/multi-indexes/btree-rtree-ngram-intersect.aql
A
asterix-app/src/test/resources/optimizerts/queries/multi-indexes/two-2nd-btree-intersect.aql
A
asterix-app/src/test/resources/optimizerts/queries/multi-indexes/two-inverted-index-intersect.aql
A
asterix-app/src/test/resources/optimizerts/queries/multi-indexes/two-rtree-intersect.aql
A
asterix-app/src/test/resources/optimizerts/queries/multi-indexes/with-primary-index-intersect.aql
A
asterix-app/src/test/resources/optimizerts/results/multi-indexes/btree-rtree-ngram-intersect.plan
A
asterix-app/src/test/resources/optimizerts/results/multi-indexes/two-2nd-btree-intersect.plan
A
asterix-app/src/test/resources/optimizerts/results/multi-indexes/two-inverted-index-intersect.plan
A
asterix-app/src/test/resources/optimizerts/results/multi-indexes/two-rtree-intersect.plan
A
asterix-app/src/test/resources/optimizerts/results/multi-indexes/with-primary-index-intersect.plan
21 files changed, 634 insertions(+), 88 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/78/578/1
diff --git
a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceUnionRule.java
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceUnionRule.java
index c446f3c..090fde7 100644
---
a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceUnionRule.java
+++
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceUnionRule.java
@@ -109,6 +109,7 @@
List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> varMap
= new ArrayList<Triple<LogicalVariable, LogicalVariable, LogicalVariable>>(
1);
+ //? why only the first one?
Triple<LogicalVariable, LogicalVariable, LogicalVariable> triple = new
Triple<LogicalVariable, LogicalVariable, LogicalVariable>(
input1Var.get(0), input2Var.get(0),
assignUnion.getVariables().get(0));
varMap.add(triple);
diff --git
a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
index 85c7478..cb9e167 100644
---
a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
+++
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
@@ -42,6 +42,7 @@
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import
org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator;
@@ -234,6 +235,11 @@
}
@Override
+ public Void visitIntersectOperator(IntersectOperator op, Void arg)
throws AlgebricksException {
+ return null;
+ }
+
+ @Override
public Void visitUnnestOperator(UnnestOperator op, Void arg) throws
AlgebricksException {
return null;
}
diff --git
a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
index 70b6770..6b64502 100644
---
a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
+++
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
@@ -129,14 +129,21 @@
* Simply picks the first index that it finds. TODO: Improve this decision
* process by making it more systematic.
*/
- protected Pair<IAccessMethod, Index> chooseIndex(Map<IAccessMethod,
AccessMethodAnalysisContext> analyzedAMs) {
+ protected Pair<IAccessMethod, Index> chooseBestIndex(Map<IAccessMethod,
AccessMethodAnalysisContext> analyzedAMs) {
+ List<Pair<IAccessMethod, Index>> list = chooseAllIndex(analyzedAMs);
+ return list.isEmpty() ? null : list.get(0);
+ }
+
+ protected List<Pair<IAccessMethod, Index>> chooseAllIndex(
+ Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs) {
+ List<Pair<IAccessMethod, Index>> result = new
ArrayList<Pair<IAccessMethod, Index>>();
Iterator<Map.Entry<IAccessMethod, AccessMethodAnalysisContext>> amIt =
analyzedAMs.entrySet().iterator();
while (amIt.hasNext()) {
Map.Entry<IAccessMethod, AccessMethodAnalysisContext> amEntry =
amIt.next();
AccessMethodAnalysisContext analysisCtx = amEntry.getValue();
Iterator<Map.Entry<Index, List<Pair<Integer, Integer>>>> indexIt =
analysisCtx.indexExprsAndVars.entrySet()
.iterator();
- if (indexIt.hasNext()) {
+ while (indexIt.hasNext()) {
Map.Entry<Index, List<Pair<Integer, Integer>>> indexEntry =
indexIt.next();
// To avoid a case where the chosen access method and a chosen
// index type is different.
@@ -149,24 +156,24 @@
// LENGTH_PARTITIONED_NGRAM_INVIX]
IAccessMethod chosenAccessMethod = amEntry.getKey();
Index chosenIndex = indexEntry.getKey();
- boolean isKeywordOrNgramIndexChosen = false;
- if (chosenIndex.getIndexType() ==
IndexType.LENGTH_PARTITIONED_WORD_INVIX
+ boolean isKeywordOrNgramIndexChosen =
+ chosenIndex.getIndexType() ==
IndexType.LENGTH_PARTITIONED_WORD_INVIX
|| chosenIndex.getIndexType() ==
IndexType.LENGTH_PARTITIONED_NGRAM_INVIX
|| chosenIndex.getIndexType() ==
IndexType.SINGLE_PARTITION_WORD_INVIX
- || chosenIndex.getIndexType() ==
IndexType.SINGLE_PARTITION_NGRAM_INVIX)
- isKeywordOrNgramIndexChosen = true;
- if ((chosenAccessMethod == BTreeAccessMethod.INSTANCE &&
chosenIndex.getIndexType() != IndexType.BTREE)
+ || chosenIndex.getIndexType() ==
IndexType.SINGLE_PARTITION_NGRAM_INVIX;
+
+ if ((chosenAccessMethod == BTreeAccessMethod.INSTANCE &&
chosenIndex.getIndexType() == IndexType.BTREE)
|| (chosenAccessMethod == RTreeAccessMethod.INSTANCE
- && chosenIndex.getIndexType() !=
IndexType.RTREE)
- || (chosenAccessMethod ==
InvertedIndexAccessMethod.INSTANCE && !isKeywordOrNgramIndexChosen)) {
- continue;
+ && chosenIndex.getIndexType() ==
IndexType.RTREE)
+ || (chosenAccessMethod ==
InvertedIndexAccessMethod.INSTANCE && isKeywordOrNgramIndexChosen)) {
+ result.add(new Pair<IAccessMethod,
Index>(chosenAccessMethod, chosenIndex));
}
- return new Pair<IAccessMethod, Index>(chosenAccessMethod,
chosenIndex);
}
}
- return null;
+ return result;
}
+
/**
* Removes irrelevant access methods candidates, based on whether the
* expressions in the query match those in the index. For example, some
diff --git
a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
index 9f95c85..fe94fa4 100644
---
a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
+++
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
@@ -59,9 +59,11 @@
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import
org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import
org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
/**
* Class for helping rewrite rules to choose and apply BTree indexes.
@@ -123,7 +125,7 @@
IOptimizationContext context) throws AlgebricksException {
SelectOperator select = (SelectOperator) selectRef.getValue();
Mutable<ILogicalExpression> conditionRef = select.getCondition();
- ILogicalOperator primaryIndexUnnestOp =
createSecondaryToPrimaryPlan(selectRef, conditionRef, subTree, null,
+ ILogicalOperator primaryIndexUnnestOp =
createSecondaryToPrimaryPlan(conditionRef, subTree, null,
chosenIndex, analysisCtx, false, false, false, context);
if (primaryIndexUnnestOp == null) {
return false;
@@ -188,7 +190,7 @@
newNullPlaceHolderVar =
indexSubTree.getDataSourceVariables().get(0);
}
- ILogicalOperator primaryIndexUnnestOp =
createSecondaryToPrimaryPlan(joinRef, conditionRef, indexSubTree,
+ ILogicalOperator primaryIndexUnnestOp =
createSecondaryToPrimaryPlan(conditionRef, indexSubTree,
probeSubTree, chosenIndex, analysisCtx, true, isLeftOuterJoin,
true, context);
if (primaryIndexUnnestOp == null) {
return false;
@@ -214,8 +216,8 @@
return true;
}
- private ILogicalOperator
createSecondaryToPrimaryPlan(Mutable<ILogicalOperator> topOpRef,
- Mutable<ILogicalExpression> conditionRef,
OptimizableOperatorSubTree indexSubTree,
+ @Override
+ public ILogicalOperator
createSecondaryToPrimaryPlan(Mutable<ILogicalExpression> conditionRef,
OptimizableOperatorSubTree indexSubTree,
OptimizableOperatorSubTree probeSubTree, Index chosenIndex,
AccessMethodAnalysisContext analysisCtx,
boolean retainInput, boolean retainNull, boolean
requiresBroadcast, IOptimizationContext context)
throws AlgebricksException {
@@ -474,7 +476,8 @@
// Assign operator that sets the constant secondary-index
search-key fields if necessary.
AssignOperator assignConstantSearchKeys = new
AssignOperator(assignKeyVarList, assignKeyExprList);
// Input to this assign is the EmptyTupleSource (which the
dataSourceScan also must have had as input).
-
assignConstantSearchKeys.getInputs().add(dataSourceOp.getInputs().get(0));
+ assignConstantSearchKeys.getInputs().add(new
MutableObject<ILogicalOperator>(
+
OperatorManipulationUtil.deepCopyWithExcutionMode(dataSourceOp.getInputs().get(0).getValue())));
assignConstantSearchKeys.setExecutionMode(dataSourceOp.getExecutionMode());
inputOp = assignConstantSearchKeys;
} else {
@@ -493,15 +496,10 @@
ExternalDataLookupOperator externalDataAccessOp =
AccessMethodUtils.createExternalDataLookupUnnestMap(
dataSourceOp, dataset, recordType, secondaryIndexUnnestOp,
context, chosenIndex, retainInput,
retainNull);
- indexSubTree.dataSourceRef.setValue(externalDataAccessOp);
return externalDataAccessOp;
} else if (!isPrimaryIndex) {
primaryIndexUnnestOp =
AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceOp, dataset, recordType,
secondaryIndexUnnestOp, context, true, retainInput,
retainNull, false);
-
- // Replace the datasource scan with the new plan rooted at
- // primaryIndexUnnestMap.
- indexSubTree.dataSourceRef.setValue(primaryIndexUnnestOp);
// Adds equivalence classes --- one equivalent class between a
primary key
// variable and a record field-access expression.
@@ -682,4 +680,15 @@
// No additional analysis required for BTrees.
return true;
}
+
+ @Override
+ public String getName() {
+ return "BTREE_ACCESS_METHOD";
+ }
+
+ @Override
+ public int compareTo(IAccessMethod o) {
+ return this.getName().compareTo(o.getName());
+ }
+
}
diff --git
a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IAccessMethod.java
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IAccessMethod.java
index 432d933..5691d57 100644
---
a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IAccessMethod.java
+++
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IAccessMethod.java
@@ -23,12 +23,14 @@
import org.apache.asterix.metadata.entities.Index;
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import
org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
import
org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import org.apache.hyracks.data.std.api.IDataOutputProvider;
/**
* Interface that an access method should implement to work with the rewrite
@@ -36,7 +38,7 @@
* methods for analyzing a select/join condition, and for rewriting the plan
* with a given index.
*/
-public interface IAccessMethod {
+public interface IAccessMethod extends Comparable<IAccessMethod>{
/**
* @return A list of function identifiers that are optimizable by this
@@ -51,7 +53,7 @@
* optimizable by this access method based on its function identifier. If
* funcExpr has been found to be optimizable, this method adds an
* OptimizableFunction to analysisCtx.matchedFuncExprs for further
analysis.
- *
+ *
* @return true if funcExpr is optimizable by this access method, false
* otherwise
* @throws AlgebricksException
@@ -63,7 +65,7 @@
/**
* Indicates whether all index expressions must be matched in order for
this
* index to be applicable.
- *
+ *
* @return boolean
*/
public boolean matchAllIndexExprs();
@@ -71,7 +73,7 @@
/**
* Indicates whether this index is applicable if only a prefix of the index
* expressions are matched.
- *
+ *
* @return boolean
*/
public boolean matchPrefixIndexExprs();
@@ -82,6 +84,15 @@
public boolean applySelectPlanTransformation(Mutable<ILogicalOperator>
selectRef,
OptimizableOperatorSubTree subTree, Index chosenIndex,
AccessMethodAnalysisContext analysisCtx,
IOptimizationContext context) throws AlgebricksException;
+
+ public ILogicalOperator
createSecondaryToPrimaryPlan(Mutable<ILogicalExpression> conditionRef,
+ OptimizableOperatorSubTree indexSubTree,
+ OptimizableOperatorSubTree probeSubTree,
+ Index chosenIndex,
+ AccessMethodAnalysisContext analysisCtx,
+ boolean retainInput, boolean retainNull, boolean requiresBroadcast,
+ IOptimizationContext context)
+ throws AlgebricksException;
/**
* Applies the plan transformation to use chosenIndex to optimize a join
query.
@@ -95,9 +106,11 @@
/**
* Analyzes expr to see whether it is optimizable by the given concrete
index.
- *
+ *
* @throws AlgebricksException
*/
public boolean exprIsOptimizable(Index index, IOptimizableFuncExpr
optFuncExpr) throws AlgebricksException;
+ public String getName();
+
}
diff --git
a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java
index 44314ff..79417a6 100644
---
a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java
+++
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java
@@ -186,7 +186,7 @@
// For the case of left-outer-join, we have to use indexes from the
inner branch.
// For the inner-join, we try to use the indexes from the inner branch
first.
// If no index is available, then we use the indexes from the outer
branch.
- Pair<IAccessMethod, Index> chosenIndex = chooseIndex(analyzedAMs);
+ Pair<IAccessMethod, Index> chosenIndex = chooseBestIndex(analyzedAMs);
if (chosenIndex == null) {
context.addToDontApplySet(this, join);
return false;
diff --git
a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceSelectAccessMethodRule.java
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceSelectAccessMethodRule.java
index 40e8712..bb849f0 100644
---
a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceSelectAccessMethodRule.java
+++
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceSelectAccessMethodRule.java
@@ -18,13 +18,17 @@
*/
package org.apache.asterix.optimizer.rules.am;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
+import java.util.TreeMap;
import org.apache.asterix.metadata.declared.AqlMetadataProvider;
import org.apache.asterix.metadata.entities.Index;
import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
@@ -32,10 +36,15 @@
import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
import
org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
import
org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import
org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import
org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractScanOperator;
+import
org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator;
+import
org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import org.apache.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
@@ -94,7 +103,7 @@
}
// Analyze select condition.
- Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs = new
HashMap<IAccessMethod, AccessMethodAnalysisContext>();
+ Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs = new
TreeMap<IAccessMethod, AccessMethodAnalysisContext>();
if (!analyzeCondition(selectCond, subTree.assignsAndUnnests,
analyzedAMs, context, typeEnvironment)) {
return false;
}
@@ -108,16 +117,15 @@
pruneIndexCandidates(analyzedAMs, context, typeEnvironment);
// Choose index to be applied.
- Pair<IAccessMethod, Index> chosenIndex = chooseIndex(analyzedAMs);
- if (chosenIndex == null) {
+ List<Pair<IAccessMethod, Index>> chosenIndexes =
chooseAllIndex(analyzedAMs);
+ if (chosenIndexes == null || chosenIndexes.size() == 0) {
context.addToDontApplySet(this, select);
return false;
}
// Apply plan transformation using chosen index.
- AccessMethodAnalysisContext analysisCtx =
analyzedAMs.get(chosenIndex.first);
- boolean res =
chosenIndex.first.applySelectPlanTransformation(selectRef, subTree,
chosenIndex.second,
- analysisCtx, context);
+ boolean res = intersectAllSecondaryIndexes(chosenIndexes, analyzedAMs,
context);
+
if (res) {
OperatorPropertiesUtil.typeOpRec(opRef, context);
}
@@ -125,6 +133,80 @@
return res;
}
+ private boolean intersectAllSecondaryIndexes(List<Pair<IAccessMethod,
Index>> chosenIndexes,
+ Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs,
IOptimizationContext context)
+ throws AlgebricksException {
+ Pair<IAccessMethod, Index> chosenIndex = null;
+ Optional<Pair<IAccessMethod, Index>> primaryIndex =
chosenIndexes.stream()
+ .filter(pair -> pair.second.isPrimaryIndex()).findFirst();
+ if (chosenIndexes.size() == 1) {
+ chosenIndex = chosenIndexes.get(0);
+ } else if (primaryIndex.isPresent()) {
+ // one primary + 2nd indexes, choose primary index directly.
+ chosenIndex = primaryIndex.get();
+ }
+ if (chosenIndex != null) {
+ AccessMethodAnalysisContext analysisCtx =
analyzedAMs.get(chosenIndex.first);
+ return chosenIndex.first
+ .applySelectPlanTransformation(selectRef, subTree,
chosenIndex.second, analysisCtx, context);
+ }
+
+ // Intersect all secondary indexes, and postpone the primary index
search.
+ Mutable<ILogicalExpression> conditionRef = select.getCondition();
+
+ List<ILogicalOperator> subRoots = new ArrayList<>();
+ for (Pair<IAccessMethod, Index> pair : chosenIndexes) {
+ AccessMethodAnalysisContext analysisCtx =
analyzedAMs.get(pair.first);
+ subRoots.add(pair.first
+ .createSecondaryToPrimaryPlan(conditionRef, subTree, null,
pair.second, analysisCtx, true, true,
+ false, context));
+ }
+ ILogicalOperator primaryUnnest =
connectAllSubRootsWithIntersect(subRoots, context);
+
+ subTree.dataSourceRef.setValue(primaryUnnest);
+ return primaryUnnest != null;
+ }
+
+ private ILogicalOperator
connectAllSubRootsWithIntersect(List<ILogicalOperator> subRoots,
+ IOptimizationContext context) throws AlgebricksException {
+ ILogicalOperator lop = subRoots.get(0);
+ List<List<LogicalVariable>> inputVars = new
ArrayList<>(subRoots.size());
+ for (int i = 0; i < subRoots.size(); i++) {
+ if (lop.getOperatorTag() != subRoots.get(i).getOperatorTag()) {
+ throw new AlgebricksException("The data source root should be
the same");
+ }
+ if (lop.getInputs().size() != 1) {
+ throw new AlgebricksException("The primary search has multiple
input");
+ }
+ if (lop.getInputs().get(0).getValue().getOperatorTag() !=
LogicalOperatorTag.ORDER) {
+ throw new AlgebricksException("The input " + i + " doesn't get
ordered");
+ }
+ ILogicalOperator curRoot = subRoots.get(i);
+ OrderOperator order = (OrderOperator)
curRoot.getInputs().get(0).getValue();
+ List<LogicalVariable> orderedColumn = new
ArrayList<>(order.getOrderExpressions().size());
+ for (Pair<OrderOperator.IOrder, Mutable<ILogicalExpression>>
orderExpression : order
+ .getOrderExpressions()) {
+ if (orderExpression.second.getValue().getExpressionTag() !=
LogicalExpressionTag.VARIABLE) {
+ throw new AlgebricksException("It should not happen, the
order by expression is not variables");
+ }
+ VariableReferenceExpression orderedVar =
(VariableReferenceExpression) orderExpression.second
+ .getValue();
+ orderedColumn.add(orderedVar.getVariableReference());
+ }
+ inputVars.add(orderedColumn);
+ }
+
+ List<LogicalVariable> outputVar = inputVars.get(0);
+ IntersectOperator intersect = new IntersectOperator(outputVar,
inputVars);
+ for (ILogicalOperator secondarySearch : subRoots) {
+ intersect.getInputs().add(secondarySearch.getInputs().get(0));
+ }
+ context.computeAndSetTypeEnvironmentForOperator(intersect);
+
intersect.setExecutionMode(AbstractLogicalOperator.ExecutionMode.LOCAL);
+ lop.getInputs().set(0, new MutableObject<>(intersect));
+ return lop;
+ }
+
protected boolean matchesOperatorPattern(Mutable<ILogicalOperator> opRef,
IOptimizationContext context) {
// First check that the operator is a select and its condition is a
function call.
AbstractLogicalOperator op1 = (AbstractLogicalOperator)
opRef.getValue();
diff --git
a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
index 35b4075..41a185b 100644
---
a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
+++
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
@@ -66,6 +66,7 @@
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import
org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
@@ -74,6 +75,7 @@
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.LogicalOperatorDeepCopyWithNewVariablesVisitor;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import
org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
import
org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexSearchModifierFactory;
import
org.apache.hyracks.storage.am.lsm.invertedindex.search.ConjunctiveEditDistanceSearchModifierFactory;
import
org.apache.hyracks.storage.am.lsm.invertedindex.search.ConjunctiveListEditDistanceSearchModifierFactory;
@@ -358,10 +360,14 @@
return false;
}
- private ILogicalOperator
createSecondaryToPrimaryPlan(OptimizableOperatorSubTree indexSubTree,
- OptimizableOperatorSubTree probeSubTree, Index chosenIndex,
IOptimizableFuncExpr optFuncExpr,
+ @Override
+ public ILogicalOperator
createSecondaryToPrimaryPlan(Mutable<ILogicalExpression> conditionRef,
+ OptimizableOperatorSubTree indexSubTree,
OptimizableOperatorSubTree probeSubTree, Index chosenIndex,
+ AccessMethodAnalysisContext analysisCtx,
boolean retainInput, boolean retainNull, boolean
requiresBroadcast, IOptimizationContext context)
throws AlgebricksException {
+
+ IOptimizableFuncExpr optFuncExpr =
AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx);
Dataset dataset = indexSubTree.dataset;
ARecordType recordType = indexSubTree.recordType;
// we made sure indexSubTree has datasource scan
@@ -389,7 +395,8 @@
// Assign operator that sets the secondary-index search-key fields.
inputOp = new AssignOperator(keyVarList, keyExprList);
// Input to this assign is the EmptyTupleSource (which the
dataSourceScan also must have had as input).
- inputOp.getInputs().add(dataSourceScan.getInputs().get(0));
+ inputOp.getInputs().add(new MutableObject<>(
+
OperatorManipulationUtil.deepCopyWithExcutionMode(dataSourceScan.getInputs().get(0).getValue())));
inputOp.setExecutionMode(dataSourceScan.getExecutionMode());
} else {
// We are optimizing a join. Add the input variable to the
secondaryIndexFuncArgs.
@@ -428,8 +435,7 @@
public boolean applySelectPlanTransformation(Mutable<ILogicalOperator>
selectRef,
OptimizableOperatorSubTree subTree, Index chosenIndex,
AccessMethodAnalysisContext analysisCtx,
IOptimizationContext context) throws AlgebricksException {
- IOptimizableFuncExpr optFuncExpr =
AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx);
- ILogicalOperator indexPlanRootOp =
createSecondaryToPrimaryPlan(subTree, null, chosenIndex, optFuncExpr, false,
+ ILogicalOperator indexPlanRootOp = createSecondaryToPrimaryPlan(null,
subTree, null, chosenIndex, analysisCtx, false,
false, false, context);
// Replace the datasource scan with the new plan rooted at
primaryIndexUnnestMap.
subTree.dataSourceRef.setValue(indexPlanRootOp);
@@ -513,8 +519,8 @@
probeSubTree.root = newProbeRootRef.getValue();
}
// Create regular indexed-nested loop join path.
- ILogicalOperator indexPlanRootOp =
createSecondaryToPrimaryPlan(indexSubTree, probeSubTree, chosenIndex,
- optFuncExpr, true, isLeftOuterJoin, true, context);
+ ILogicalOperator indexPlanRootOp = createSecondaryToPrimaryPlan(null,
indexSubTree, probeSubTree, chosenIndex,
+ analysisCtx, true, isLeftOuterJoin, true, context);
indexSubTree.dataSourceRef.setValue(indexPlanRootOp);
// Change join into a select with the same condition.
@@ -1173,4 +1179,14 @@
}
context.computeAndSetTypeEnvironmentForOperator(op);
}
+
+ @Override
+ public String getName() {
+ return "INVERTED_INDEX_ACCESS_METHOD";
+ }
+
+ @Override
+ public int compareTo(IAccessMethod o) {
+ return this.getName().compareTo(o.getName());
+ }
}
diff --git
a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
index 4aebbda..760608f 100644
---
a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
+++
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
@@ -54,6 +54,7 @@
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import
org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
/**
* Class for helping rewrite rules to choose and apply RTree indexes.
@@ -77,8 +78,8 @@
public boolean analyzeFuncExprArgs(AbstractFunctionCallExpression funcExpr,
List<AbstractLogicalOperator> assignsAndUnnests,
AccessMethodAnalysisContext analysisCtx,
IOptimizationContext context, IVariableTypeEnvironment
typeEnvironment) throws AlgebricksException {
- boolean matches =
AccessMethodUtils.analyzeFuncExprArgsForOneConstAndVar(funcExpr, analysisCtx,
context,
- typeEnvironment);
+ boolean matches = AccessMethodUtils
+ .analyzeFuncExprArgsForOneConstAndVar(funcExpr, analysisCtx,
context, typeEnvironment);
if (!matches) {
matches =
AccessMethodUtils.analyzeFuncExprArgsForTwoVars(funcExpr, analysisCtx);
}
@@ -100,15 +101,23 @@
OptimizableOperatorSubTree subTree, Index chosenIndex,
AccessMethodAnalysisContext analysisCtx,
IOptimizationContext context) throws AlgebricksException {
// TODO: We can probably do something smarter here based on
selectivity or MBR area.
- IOptimizableFuncExpr optFuncExpr =
AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx);
- ILogicalOperator primaryIndexUnnestOp =
createSecondaryToPrimaryPlan(subTree, null, chosenIndex, optFuncExpr,
- analysisCtx, false, false, false, context);
+ ILogicalOperator primaryIndexUnnestOp =
createSecondaryToPrimaryPlan(subTree, null, chosenIndex, analysisCtx,
+ false, false, false, context);
if (primaryIndexUnnestOp == null) {
return false;
}
// Replace the datasource scan with the new plan rooted at
primaryIndexUnnestMap.
subTree.dataSourceRef.setValue(primaryIndexUnnestOp);
return true;
+ }
+
+ @Override
+ public ILogicalOperator
createSecondaryToPrimaryPlan(Mutable<ILogicalExpression> conditionRef,
+ OptimizableOperatorSubTree indexSubTree,
OptimizableOperatorSubTree probeSubTree, Index chosenIndex,
+ AccessMethodAnalysisContext analysisCtx, boolean retainInput,
boolean retainNull, boolean requiresBroadcast,
+ IOptimizationContext context) throws AlgebricksException {
+ return createSecondaryToPrimaryPlan(indexSubTree, probeSubTree,
chosenIndex, analysisCtx, retainInput,
+ retainNull, requiresBroadcast, context);
}
@Override
@@ -121,12 +130,12 @@
// Determine probe and index subtrees based on chosen index.
OptimizableOperatorSubTree indexSubTree = null;
OptimizableOperatorSubTree probeSubTree = null;
- if (!isLeftOuterJoin && leftSubTree.hasDataSourceScan()
- &&
dataset.getDatasetName().equals(leftSubTree.dataset.getDatasetName())) {
+ if (!isLeftOuterJoin && leftSubTree.hasDataSourceScan() &&
dataset.getDatasetName()
+ .equals(leftSubTree.dataset.getDatasetName())) {
indexSubTree = leftSubTree;
probeSubTree = rightSubTree;
- } else if (rightSubTree.hasDataSourceScan()
- &&
dataset.getDatasetName().equals(rightSubTree.dataset.getDatasetName())) {
+ } else if (rightSubTree.hasDataSourceScan() && dataset.getDatasetName()
+ .equals(rightSubTree.dataset.getDatasetName())) {
indexSubTree = rightSubTree;
probeSubTree = leftSubTree;
}
@@ -143,9 +152,8 @@
}
// TODO: We can probably do something smarter here based on
selectivity or MBR area.
- IOptimizableFuncExpr optFuncExpr =
AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx);
ILogicalOperator primaryIndexUnnestOp =
createSecondaryToPrimaryPlan(indexSubTree, probeSubTree, chosenIndex,
- optFuncExpr, analysisCtx, true, isLeftOuterJoin, true,
context);
+ analysisCtx, true, isLeftOuterJoin, true, context);
if (primaryIndexUnnestOp == null) {
return false;
}
@@ -168,9 +176,11 @@
}
private ILogicalOperator
createSecondaryToPrimaryPlan(OptimizableOperatorSubTree indexSubTree,
- OptimizableOperatorSubTree probeSubTree, Index chosenIndex,
IOptimizableFuncExpr optFuncExpr,
- AccessMethodAnalysisContext analysisCtx, boolean retainInput,
boolean retainNull, boolean requiresBroadcast,
- IOptimizationContext context) throws AlgebricksException {
+ OptimizableOperatorSubTree probeSubTree, Index chosenIndex,
AccessMethodAnalysisContext analysisCtx,
+ boolean retainInput, boolean retainNull, boolean
requiresBroadcast, IOptimizationContext context)
+ throws AlgebricksException {
+
+ IOptimizableFuncExpr optFuncExpr =
AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx);
Dataset dataset = indexSubTree.dataset;
ARecordType recordType = indexSubTree.recordType;
@@ -196,8 +206,8 @@
ArrayList<LogicalVariable> keyVarList = new
ArrayList<LogicalVariable>();
// List of expressions for the assign.
ArrayList<Mutable<ILogicalExpression>> keyExprList = new
ArrayList<Mutable<ILogicalExpression>>();
- Pair<ILogicalExpression, Boolean> returnedSearchKeyExpr =
AccessMethodUtils.createSearchKeyExpr(optFuncExpr,
- indexSubTree, probeSubTree);
+ Pair<ILogicalExpression, Boolean> returnedSearchKeyExpr =
AccessMethodUtils
+ .createSearchKeyExpr(optFuncExpr, indexSubTree, probeSubTree);
ILogicalExpression searchKeyExpr = returnedSearchKeyExpr.first;
for (int i = 0; i < numSecondaryKeys; i++) {
@@ -224,25 +234,28 @@
if (probeSubTree == null) {
// We are optimizing a selection query.
// Input to this assign is the EmptyTupleSource (which the
dataSourceScan also must have had as input).
- assignSearchKeys.getInputs().add(dataSourceOp.getInputs().get(0));
+ assignSearchKeys.getInputs().add(new
MutableObject<ILogicalOperator>(
+
OperatorManipulationUtil.deepCopyWithExcutionMode(dataSourceOp.getInputs().get(0).getValue())));
assignSearchKeys.setExecutionMode(dataSourceOp.getExecutionMode());
} else {
// We are optimizing a join, place the assign op top of the probe
subtree.
assignSearchKeys.getInputs().add(probeSubTree.rootRef);
}
- UnnestMapOperator secondaryIndexUnnestOp =
AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType,
- chosenIndex, assignSearchKeys, jobGenParams, context, false,
retainInput);
+ UnnestMapOperator secondaryIndexUnnestOp = AccessMethodUtils
+ .createSecondaryIndexUnnestMap(dataset, recordType,
chosenIndex, assignSearchKeys, jobGenParams,
+ context, false, retainInput);
// Generate the rest of the upstream plan which feeds the search
results into the primary index.
if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
- ExternalDataLookupOperator externalDataAccessOp =
AccessMethodUtils.createExternalDataLookupUnnestMap(
- dataSourceOp, dataset, recordType, secondaryIndexUnnestOp,
context, chosenIndex, retainInput,
- retainNull);
+ ExternalDataLookupOperator externalDataAccessOp = AccessMethodUtils
+ .createExternalDataLookupUnnestMap(dataSourceOp, dataset,
recordType, secondaryIndexUnnestOp,
+ context, chosenIndex, retainInput, retainNull);
return externalDataAccessOp;
} else {
- UnnestMapOperator primaryIndexUnnestOp =
AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceOp,
- dataset, recordType, secondaryIndexUnnestOp, context,
true, retainInput, false, false);
+ UnnestMapOperator primaryIndexUnnestOp = AccessMethodUtils
+ .createPrimaryIndexUnnestMap(dataSourceOp, dataset,
recordType, secondaryIndexUnnestOp, context,
+ true, retainInput, false, false);
return primaryIndexUnnestOp;
}
@@ -257,4 +270,15 @@
// No additional analysis required.
return true;
}
+
+ @Override
+ public String getName() {
+ return "RTREE_ACCESS_METHOD";
+ }
+
+ @Override
+ public int compareTo(IAccessMethod o) {
+ return this.getName().compareTo(o.getName());
+ }
+
}
diff --git
a/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java
b/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java
index 538e7ca..4139e69 100644
---
a/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java
+++
b/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java
@@ -57,8 +57,8 @@
private static final String EXTENSION_RESULT = "plan";
private static final String FILENAME_IGNORE = "ignore.txt";
private static final String FILENAME_ONLY = "only.txt";
- private static final String PATH_BASE = "src" + SEPARATOR + "test" +
SEPARATOR + "resources" + SEPARATOR
- + "optimizerts" + SEPARATOR;
+ private static final String PATH_BASE =
+ "src" + SEPARATOR + "test" + SEPARATOR + "resources" + SEPARATOR +
"optimizerts" + SEPARATOR;
private static final String PATH_QUERIES = PATH_BASE + "queries" +
SEPARATOR;
private static final String PATH_EXPECTED = PATH_BASE + "results" +
SEPARATOR;
private static final String PATH_ACTUAL = "opttest" + SEPARATOR;
@@ -95,26 +95,34 @@
AsterixHyracksIntegrationUtil.deinit(true);
}
- private static void suiteBuild(File dir, Collection<Object[]> testArgs,
String path) {
- for (File file : dir.listFiles()) {
- if (file.isDirectory() && !file.getName().startsWith(".")) {
- suiteBuild(file, testArgs, path + file.getName() + SEPARATOR);
+ private static void suiteBuildPerFile(File file, Collection<Object[]>
testArgs, String path) {
+ if (file.isDirectory() && !file.getName().startsWith(".")) {
+ for (File innerfile : file.listFiles()) {
+ String subdir = innerfile.isDirectory() ? path +
innerfile.getName() + SEPARATOR : path;
+ suiteBuildPerFile(innerfile, testArgs, subdir);
}
- if (file.isFile() && file.getName().endsWith(EXTENSION_QUERY)
+ }
+ if (file.isFile() && file.getName().endsWith(EXTENSION_QUERY)
// && !ignore.contains(path + file.getName())
- ) {
- String resultFileName =
AsterixTestHelper.extToResExt(file.getName(), EXTENSION_RESULT);
- File expectedFile = new File(PATH_EXPECTED + path +
resultFileName);
- File actualFile = new File(PATH_ACTUAL + SEPARATOR +
path.replace(SEPARATOR, "_") + resultFileName);
- testArgs.add(new Object[] { file, expectedFile, actualFile });
- }
+ ) {
+ String resultFileName =
AsterixTestHelper.extToResExt(file.getName(), EXTENSION_RESULT);
+ File expectedFile = new File(PATH_EXPECTED + path +
resultFileName);
+ File actualFile = new File(PATH_ACTUAL + SEPARATOR +
path.replace(SEPARATOR, "_") + resultFileName);
+ testArgs.add(new Object[] { file, expectedFile, actualFile });
}
}
@Parameters(name = "OptimizerTest {index}: {0}")
public static Collection<Object[]> tests() {
Collection<Object[]> testArgs = new ArrayList<Object[]>();
- suiteBuild(new File(PATH_QUERIES), testArgs, "");
+ if (only.isEmpty()) {
+ suiteBuildPerFile(new File(PATH_QUERIES), testArgs, "");
+ } else {
+ for (String path : only) {
+ suiteBuildPerFile(new File(PATH_QUERIES + path), testArgs,
+ path.lastIndexOf(SEPARATOR) < 0 ? "" :
path.substring(0, path.lastIndexOf(SEPARATOR) + 1));
+ }
+ }
return testArgs;
}
@@ -131,8 +139,8 @@
@Test
public void test() throws Exception {
try {
- String queryFileShort =
queryFile.getPath().substring(PATH_QUERIES.length()).replace(SEPARATOR.charAt(0),
- '/');
+ String queryFileShort =
queryFile.getPath().substring(PATH_QUERIES.length())
+ .replace(SEPARATOR.charAt(0), '/');
if (!only.isEmpty()) {
boolean toRun = TestHelper.isInPrefixList(only,
queryFileShort);
if (!toRun) {
@@ -174,12 +182,14 @@
lineActual = readerActual.readLine();
// Assert.assertEquals(lineExpected, lineActual);
if (lineActual == null) {
- throw new Exception("Result for " + queryFile + "
changed at line " + num + ":\n< "
- + lineExpected + "\n> ");
+ throw new Exception(
+ "Result for " + queryFile + " changed at line
" + num + ":\n< " + lineExpected
+ + "\n> ");
}
if (!lineExpected.equals(lineActual)) {
- throw new Exception("Result for " + queryFile + "
changed at line " + num + ":\n< "
- + lineExpected + "\n> " + lineActual);
+ throw new Exception(
+ "Result for " + queryFile + " changed at line
" + num + ":\n< " + lineExpected + "\n> "
+ + lineActual);
}
++num;
}
diff --git
a/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java
b/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java
index 199ae39..32533d5 100644
---
a/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java
+++
b/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java
@@ -52,8 +52,6 @@
protected static AsterixTransactionProperties txnProperties;
private final static TestExecutor testExecutor = new TestExecutor();
- protected static TestGroup FailedGroup;
-
@BeforeClass
public static void setUp() throws Exception {
File outdir = new File(PATH_ACTUAL);
@@ -93,6 +91,6 @@
@Test
public void test() throws Exception {
- testExecutor.executeTest(PATH_ACTUAL, tcCtx, null, false, FailedGroup);
+ testExecutor.executeTest(PATH_ACTUAL, tcCtx, null, false,
ExecutionTestUtil.FailedGroup);
}
}
diff --git
a/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/btree-rtree-ngram-intersect.aql
b/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/btree-rtree-ngram-intersect.aql
new file mode 100644
index 0000000..5f57d07
--- /dev/null
+++
b/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/btree-rtree-ngram-intersect.aql
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Tests three types of secondary indexes should trigger
intersection rule
+ * Success : Yes
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type tTweet as closed {
+ id: int32,
+ location: point,
+ message: string,
+ create_at: datetime,
+ misc: string
+}
+
+create dataset dsTweet(tTweet) primary key id;
+
+create index ngram_index on dsTweet(message) type ngram(3);
+create index time_index on dsTweet(create_at) type btree;
+create index location_index on dsTweet(location) type rtree;
+
+write output to nc1:"rttest/btree-rtree-ngram-intersect.adm";
+
+let $region :=
create-rectangle(create-point(-128.43007812500002,20.298506037222175),
create-point(-64.26992187500002,54.56902589732035))
+let $ts_start := datetime("2015-11-11T00:00:00Z")
+let $ts_end := datetime("2015-12-18T23:59:59Z")
+let $keyword := "hello"
+for $t in dataset dsTweet
+where $t.create_at >= $ts_start and $t.create_at < $ts_end
+ and spatial-intersect($t.location, $region)
+ and contains($t.message, $keyword)
+return $t
+
diff --git
a/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/two-2nd-btree-intersect.aql
b/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/two-2nd-btree-intersect.aql
new file mode 100644
index 0000000..f0ecf57
--- /dev/null
+++
b/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/two-2nd-btree-intersect.aql
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Tests two secondary Btree indexes should trigger
intersection rule
+ * Success : Yes
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type tTweet as closed {
+ id: int32,
+ location: point,
+ message: string,
+ create_at: datetime,
+ misc: string
+}
+
+create dataset dsTweet(tTweet) primary key id;
+
+create index time_index on dsTweet(create_at) type btree;
+create index misc_index on dsTweet(misc) type btree;
+
+write output to nc1:"rttest/two-2nd-btree-intersect.adm";
+
+let $ts_start := datetime("2015-11-11T00:00:00Z")
+let $ts_end := datetime("2015-12-18T23:59:59Z")
+let $keyword := "hello"
+for $t in dataset dsTweet
+where $t.create_at >= $ts_start and $t.create_at < $ts_end
+ and $t.misc > $keyword
+return $t
+
diff --git
a/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/two-inverted-index-intersect.aql
b/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/two-inverted-index-intersect.aql
new file mode 100644
index 0000000..2947d61
--- /dev/null
+++
b/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/two-inverted-index-intersect.aql
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Tests keyword index and the ngram indexe should trigger
intersection rule
+ * Success : Yes
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type tTweet as closed {
+ id: int32,
+ message: string,
+ create_at: datetime,
+ misc: string
+}
+
+create dataset dsTweet(tTweet) primary key id;
+
+create index msg_index on dsTweet(message) type ngram(3);
+create index msc_index on dsTweet(misc) type keyword;
+
+for $t in dataset dsTweet
+let $ed := edit-distance($t.message, "Suzanna Tilson")
+let $jacc := similarity-jaccard-check(word-tokens($t.misc), word-tokens("love
like verizon"), 0.2f)
+where $jacc[0] and $ed <= 2
+return $t
+
diff --git
a/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/two-rtree-intersect.aql
b/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/two-rtree-intersect.aql
new file mode 100644
index 0000000..40c9617
--- /dev/null
+++
b/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/two-rtree-intersect.aql
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Tests two secondary rtree indexes should trigger
intersection rule
+ * Success : Yes
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type tTweet as closed {
+ id: int32,
+ location: point,
+ place: rectangle,
+ misc: string
+}
+
+create dataset dsTweet(tTweet) primary key id;
+
+create index loc_index on dsTweet(location) type rtree;
+create index plc_index on dsTweet(place) type rtree;
+
+let $region :=
create-rectangle(create-point(-128.43007812500002,20.298506037222175),
create-point(-64.26992187500002,54.56902589732035))
+for $t in dataset dsTweet
+where spatial-intersect($t.location , $region) and spatial-intersect($t.place,
$region)
+return $t
+
diff --git
a/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/with-primary-index-intersect.aql
b/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/with-primary-index-intersect.aql
new file mode 100644
index 0000000..d90f119
--- /dev/null
+++
b/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/with-primary-index-intersect.aql
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Tests primary index will be chosen if it existed in the
job plan, no intersection should apply
+ * Success : Yes
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type tTweet as closed {
+ id: int32,
+ location: point,
+ message: string,
+ create_at: datetime,
+ misc: string
+}
+
+create dataset dsTweet(tTweet) primary key id;
+
+create index ngram_index on dsTweet(message) type ngram(3);
+create index time_index on dsTweet(create_at) type btree;
+create index location_index on dsTweet(location) type rtree;
+
+write output to nc1:"rttest/btree-rtree-ngram-intersect.adm";
+
+let $region :=
create-rectangle(create-point(-128.43007812500002,20.298506037222175),
create-point(-64.26992187500002,54.56902589732035))
+let $ts_start := datetime("2015-11-11T00:00:00Z")
+let $ts_end := datetime("2015-12-18T23:59:59Z")
+let $keyword := "hello"
+for $t in dataset dsTweet
+where $t.create_at >= $ts_start and $t.create_at < $ts_end
+ and spatial-intersect($t.location, $region)
+ and contains($t.message, $keyword)
+ and $t.id > 0
+return $t
+
diff --git
a/asterix-app/src/test/resources/optimizerts/results/multi-indexes/btree-rtree-ngram-intersect.plan
b/asterix-app/src/test/resources/optimizerts/results/multi-indexes/btree-rtree-ngram-intersect.plan
new file mode 100644
index 0000000..2d74649
--- /dev/null
+++
b/asterix-app/src/test/resources/optimizerts/results/multi-indexes/btree-rtree-ngram-intersect.plan
@@ -0,0 +1,37 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- INTERSECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$29(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$31(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- LENGTH_PARTITIONED_INVERTED_INDEX_SEARCH
|PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$40(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- RTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git
a/asterix-app/src/test/resources/optimizerts/results/multi-indexes/two-2nd-btree-intersect.plan
b/asterix-app/src/test/resources/optimizerts/results/multi-indexes/two-2nd-btree-intersect.plan
new file mode 100644
index 0000000..5adb550
--- /dev/null
+++
b/asterix-app/src/test/resources/optimizerts/results/multi-indexes/two-2nd-btree-intersect.plan
@@ -0,0 +1,28 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- INTERSECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$19(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$23(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git
a/asterix-app/src/test/resources/optimizerts/results/multi-indexes/two-inverted-index-intersect.plan
b/asterix-app/src/test/resources/optimizerts/results/multi-indexes/two-inverted-index-intersect.plan
new file mode 100644
index 0000000..7d99635
--- /dev/null
+++
b/asterix-app/src/test/resources/optimizerts/results/multi-indexes/two-inverted-index-intersect.plan
@@ -0,0 +1,26 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- INTERSECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$18(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- LENGTH_PARTITIONED_INVERTED_INDEX_SEARCH
|PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$20(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- LENGTH_PARTITIONED_INVERTED_INDEX_SEARCH
|PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git
a/asterix-app/src/test/resources/optimizerts/results/multi-indexes/two-rtree-intersect.plan
b/asterix-app/src/test/resources/optimizerts/results/multi-indexes/two-rtree-intersect.plan
new file mode 100644
index 0000000..05d43a9
--- /dev/null
+++
b/asterix-app/src/test/resources/optimizerts/results/multi-indexes/two-rtree-intersect.plan
@@ -0,0 +1,32 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- INTERSECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$24(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- RTREE_SEARCH |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- SPLIT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$33(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- RTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- SPLIT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git
a/asterix-app/src/test/resources/optimizerts/results/multi-indexes/with-primary-index-intersect.plan
b/asterix-app/src/test/resources/optimizerts/results/multi-indexes/with-primary-index-intersect.plan
new file mode 100644
index 0000000..7978f8a
--- /dev/null
+++
b/asterix-app/src/test/resources/optimizerts/results/multi-indexes/with-primary-index-intersect.plan
@@ -0,0 +1,11 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
--
To view, visit https://asterix-gerrit.ics.uci.edu/578
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ie167918fb23e39c8728840e4a90c1b85bf1bde85
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Jianfeng Jia <[email protected]>
