Jianfeng Jia has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/1727
Change subject: Change logical plan to apply filter from 2ndary index
......................................................................
Change logical plan to apply filter from 2ndary index
- Changes the IntroduceLSMComponentFilterRule to
replace the constant filter value from the query to the value
carried from 2ndary index search.
- Can use 2ndary index filter even the query doens't contain
any filter related condition.
Change-Id: I0e2fe0208662e5dcd49d1a22bfb58f96533e9497
---
M
asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
M
asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/IndexSearchPOperator.java
M
asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java
M
asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/RTreeSearchPOperator.java
M
asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
M
asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java
A
asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter/btree-btree-search-wo-query-filter.aql
A
asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter/btree-btree-search.aql
A
asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter/inverted-btree-search-wo-query-filter.aql
A
asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter/inverted-btree-search.aql
A
asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter/multi-index-btree-search-wo-query-filter.aql
A
asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter/multi-index-btree-search.aql
A
asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter/rtree-btree-search-wo-query-filter.aql
A
asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter/rtree-btree-search.aql
A
asterixdb/asterix-app/src/test/resources/optimizerts/results/filter/btree-btree-search-wo-query-filter.plan
A
asterixdb/asterix-app/src/test/resources/optimizerts/results/filter/btree-btree-search.plan
A
asterixdb/asterix-app/src/test/resources/optimizerts/results/filter/inverted-btree-search-wo-query-filter.plan
A
asterixdb/asterix-app/src/test/resources/optimizerts/results/filter/inverted-btree-search.plan
A
asterixdb/asterix-app/src/test/resources/optimizerts/results/filter/multi-index-btree-search-wo-query-filter.plan
A
asterixdb/asterix-app/src/test/resources/optimizerts/results/filter/multi-index-btree-search.plan
A
asterixdb/asterix-app/src/test/resources/optimizerts/results/filter/rtree-btree-search-wo-query-filter.plan
A
asterixdb/asterix-app/src/test/resources/optimizerts/results/filter/rtree-btree-search.plan
M
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
M
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
M
hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractScanOperator.java
M
hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractUnnestMapOperator.java
M
hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java
M
hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/AbstractRTreeOperatorTest.java
28 files changed, 744 insertions(+), 49 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/27/1727/1
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
index 9dd57d5..5468856 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
@@ -115,6 +115,7 @@
int[] minFilterFieldIndexes =
getKeyIndexes(unnestMap.getMinFilterVars(), inputSchemas);
int[] maxFilterFieldIndexes =
getKeyIndexes(unnestMap.getMaxFilterVars(), inputSchemas);
+ boolean propagateFilter = unnestMap.getPropagateIndexFilter();
MetadataProvider metadataProvider = (MetadataProvider)
context.getMetadataProvider();
Dataset dataset =
metadataProvider.findDataset(jobGenParams.getDataverseName(),
jobGenParams.getDatasetName());
@@ -124,7 +125,7 @@
Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> btreeSearch =
metadataProvider.buildBtreeRuntime(
builder.getJobSpec(), opSchema, typeEnv, context,
jobGenParams.getRetainInput(), retainMissing,
dataset, jobGenParams.getIndexName(), lowKeyIndexes,
highKeyIndexes, jobGenParams.isLowKeyInclusive(),
- jobGenParams.isHighKeyInclusive(), minFilterFieldIndexes,
maxFilterFieldIndexes);
+ jobGenParams.isHighKeyInclusive(), propagateFilter,
minFilterFieldIndexes, maxFilterFieldIndexes);
builder.contributeHyracksOperator(unnestMap, btreeSearch.first);
builder.contributeAlgebricksPartitionConstraint(btreeSearch.first,
btreeSearch.second);
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/IndexSearchPOperator.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/IndexSearchPOperator.java
index ce43480..9f46e6a 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/IndexSearchPOperator.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/IndexSearchPOperator.java
@@ -63,7 +63,7 @@
IDataSource<?> ds = idx.getDataSource();
IDataSourcePropertiesProvider dspp = ds.getPropertiesProvider();
AbstractScanOperator as = (AbstractScanOperator) op;
- deliveredProperties = dspp.computePropertiesVector(as.getVariables());
+ deliveredProperties =
dspp.computePropertiesVector(as.getScanVariables());
}
protected int[] getKeyIndexes(List<LogicalVariable> keyVarList,
IOperatorSchema[] inputSchemas) {
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java
index 3d7ba34..dd26865 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java
@@ -123,6 +123,7 @@
}
int[] keyIndexes = getKeyIndexes(jobGenParams.getKeyVarList(),
inputSchemas);
+ boolean propagateIndexFilter = unnestMapOp.getPropagateIndexFilter();
int[] minFilterFieldIndexes =
getKeyIndexes(unnestMapOp.getMinFilterVars(), inputSchemas);
int[] maxFilterFieldIndexes =
getKeyIndexes(unnestMapOp.getMaxFilterVars(), inputSchemas);
boolean retainNull = false;
@@ -136,7 +137,8 @@
jobGenParams.getRetainInput(), retainNull,
jobGenParams.getDatasetName(), dataset,
jobGenParams.getIndexName(),
jobGenParams.getSearchKeyType(), keyIndexes,
jobGenParams.getSearchModifierType(),
jobGenParams.getSimilarityThreshold(),
- minFilterFieldIndexes, maxFilterFieldIndexes,
jobGenParams.getIsFullTextSearch());
+ propagateIndexFilter, minFilterFieldIndexes,
maxFilterFieldIndexes,
+ jobGenParams.getIsFullTextSearch());
// Contribute operator in hyracks job.
builder.contributeHyracksOperator(unnestMapOp, invIndexSearch.first);
@@ -150,8 +152,8 @@
AbstractUnnestMapOperator unnestMap, IOperatorSchema opSchema,
boolean retainInput, boolean retainMissing,
String datasetName, Dataset dataset, String indexName, ATypeTag
searchKeyType, int[] keyFields,
SearchModifierType searchModifierType, IAlgebricksConstantValue
similarityThreshold,
- int[] minFilterFieldIndexes, int[] maxFilterFieldIndexes, boolean
isFullTextSearchQuery)
- throws AlgebricksException {
+ boolean propagateIndexFilter, int[] minFilterFieldIndexes, int[]
maxFilterFieldIndexes,
+ boolean isFullTextSearchQuery) throws AlgebricksException {
try {
IAObject simThresh = ((AsterixConstantValue)
similarityThreshold).getObject();
IAType itemType =
MetadataManager.INSTANCE.getDatatype(metadataProvider.getMetadataTxnContext(),
@@ -260,7 +262,7 @@
dataset.getSearchCallbackFactory(metadataProvider.getStorageComponentProvider(),
secondaryIndex,
((JobEventListenerFactory)
jobSpec.getJobletEventListenerFactory()).getJobId(),
IndexOperation.SEARCH, null),
- minFilterFieldIndexes, maxFilterFieldIndexes,
+ propagateIndexFilter, minFilterFieldIndexes,
maxFilterFieldIndexes,
metadataProvider.getStorageComponentProvider().getMetadataPageManagerFactory(),
isFullTextSearchQuery);
return new Pair<>(invIndexSearchOp,
secondarySplitsAndConstraint.second);
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/RTreeSearchPOperator.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/RTreeSearchPOperator.java
index f9d4c80..ff26c9a 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/RTreeSearchPOperator.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/RTreeSearchPOperator.java
@@ -66,7 +66,7 @@
@Override
public void contributeRuntimeOperator(IHyracksJobBuilder builder,
JobGenContext context, ILogicalOperator op,
IOperatorSchema opSchema, IOperatorSchema[] inputSchemas,
IOperatorSchema outerPlanSchema)
- throws AlgebricksException {
+ throws AlgebricksException {
AbstractUnnestMapOperator unnestMap = (AbstractUnnestMapOperator) op;
ILogicalExpression unnestExpr =
unnestMap.getExpressionRef().getValue();
if (unnestExpr.getExpressionTag() !=
LogicalExpressionTag.FUNCTION_CALL) {
@@ -81,6 +81,7 @@
jobGenParams.readFromFuncArgs(unnestFuncExpr.getArguments());
int[] keyIndexes = getKeyIndexes(jobGenParams.getKeyVarList(),
inputSchemas);
+ boolean propagateIndexFilter = unnestMap.getPropagateIndexFilter();
int[] minFilterFieldIndexes =
getKeyIndexes(unnestMap.getMinFilterVars(), inputSchemas);
int[] maxFilterFieldIndexes =
getKeyIndexes(unnestMap.getMaxFilterVars(), inputSchemas);
@@ -97,9 +98,10 @@
// By nature, LEFT_OUTER_UNNEST_MAP should generate null values
for non-matching tuples.
retainNull = true;
}
- Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> rtreeSearch =
mp.buildRtreeRuntime(
- builder.getJobSpec(), outputVars, opSchema, typeEnv, context,
jobGenParams.getRetainInput(), retainNull,
- dataset, jobGenParams.getIndexName(), keyIndexes,
minFilterFieldIndexes, maxFilterFieldIndexes);
+ Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> rtreeSearch =
+ mp.buildRtreeRuntime(builder.getJobSpec(), outputVars,
opSchema, typeEnv, context,
+ jobGenParams.getRetainInput(), retainNull, dataset,
jobGenParams.getIndexName(), keyIndexes,
+ propagateIndexFilter, minFilterFieldIndexes,
maxFilterFieldIndexes);
builder.contributeHyracksOperator(unnestMap, rtreeSearch.first);
builder.contributeAlgebricksPartitionConstraint(rtreeSearch.first,
rtreeSearch.second);
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
index e059574..2c5e358 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
@@ -355,6 +355,7 @@
sourceVars = ((AbstractUnnestMapOperator) unnestMapOp).getVariables();
// Assumes the primary keys are located at the end.
+ // TODO look at here about the variables
int start = sourceVars.size() - numPrimaryKeys;
int stop = sourceVars.size();
for (int i = start; i < stop; i++) {
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java
index 18641e5..b6d7115 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java
@@ -19,9 +19,12 @@
package org.apache.asterix.optimizer.rules.am;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Queue;
+import java.util.logging.Logger;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.metadata.declared.DataSource;
@@ -63,6 +66,8 @@
public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule {
+ static final Logger LOGGER =
Logger.getLogger(IntroduceLSMComponentFilterRule.class.getName());
+
protected IVariableTypeEnvironment typeEnvironment = null;
@Override
@@ -80,12 +85,6 @@
}
AbstractLogicalOperator op = (AbstractLogicalOperator)
opRef.getValue();
- typeEnvironment = context.getOutputTypeEnvironment(op);
- ILogicalExpression condExpr = ((SelectOperator)
op).getCondition().getValue();
- AccessMethodAnalysisContext analysisCtx = analyzeCondition(condExpr,
context, typeEnvironment);
- if (analysisCtx.getMatchedFuncExprs().isEmpty()) {
- return false;
- }
Dataset dataset = getDataset(op, context);
List<String> filterFieldName = null;
@@ -101,22 +100,33 @@
if (filterFieldName == null || recType == null) {
return false;
}
- List<Index> datasetIndexes = ((MetadataProvider)
context.getMetadataProvider())
- .getDatasetIndexes(dataset.getDataverseName(),
dataset.getDatasetName());
+
+ IAType filterType = recType.getSubFieldType(filterFieldName);
+
+ typeEnvironment = context.getOutputTypeEnvironment(op);
+ ILogicalExpression condExpr = ((SelectOperator)
op).getCondition().getValue();
+ AccessMethodAnalysisContext analysisCtx = analyzeCondition(condExpr,
context, typeEnvironment);
List<IOptimizableFuncExpr> optFuncExprs = new ArrayList<>();
- for (int i = 0; i < analysisCtx.getMatchedFuncExprs().size(); i++) {
- IOptimizableFuncExpr optFuncExpr =
analysisCtx.getMatchedFuncExpr(i);
- boolean found = findMacthedExprFieldName(optFuncExpr, op, dataset,
recType, datasetIndexes, context);
- if (found && optFuncExpr.getFieldName(0).equals(filterFieldName)) {
- optFuncExprs.add(optFuncExpr);
+ if (!analysisCtx.getMatchedFuncExprs().isEmpty()) {
+ List<Index> datasetIndexes = ((MetadataProvider)
context.getMetadataProvider())
+ .getDatasetIndexes(dataset.getDataverseName(),
dataset.getDatasetName());
+
+ for (int i = 0; i < analysisCtx.getMatchedFuncExprs().size(); i++)
{
+ IOptimizableFuncExpr optFuncExpr =
analysisCtx.getMatchedFuncExpr(i);
+ boolean found = findMacthedExprFieldName(optFuncExpr, op,
dataset, recType, datasetIndexes, context);
+ if (found &&
optFuncExpr.getFieldName(0).equals(filterFieldName)) {
+ optFuncExprs.add(optFuncExpr);
+ }
}
}
+
if (optFuncExprs.isEmpty()) {
- return false;
+ assignFilterFromSecondaryUnnest(op, dataset, context, filterType);
+ } else {
+ assignFilterFromQuery(optFuncExprs, op, dataset, context,
filterType);
}
- changePlan(optFuncExprs, op, dataset, context);
OperatorPropertiesUtil.typeOpRec(opRef, context);
context.addToDontApplySet(this, op);
@@ -129,8 +139,8 @@
List<Mutable<ILogicalExpression>> assignKeyExprList = new
ArrayList<>();
for (IOptimizableFuncExpr optFuncExpr : optFuncExprs) {
- ComparisonKind ck = AlgebricksBuiltinFunctions
-
.getComparisonType(optFuncExpr.getFuncExpr().getFunctionIdentifier());
+ ComparisonKind ck =
+
AlgebricksBuiltinFunctions.getComparisonType(optFuncExpr.getFuncExpr().getFunctionIdentifier());
ILogicalExpression searchKeyExpr = optFuncExpr.getConstantExpr(0);
LogicalVariable var = context.newVar();
assignKeyExprList.add(new
MutableObject<ILogicalExpression>(searchKeyExpr));
@@ -147,9 +157,11 @@
return new AssignOperator(assignKeyVarList, assignKeyExprList);
}
- private void changePlan(List<IOptimizableFuncExpr> optFuncExprs,
AbstractLogicalOperator op, Dataset dataset,
- IOptimizationContext context) throws AlgebricksException {
+ private void assignFilterFromQuery(List<IOptimizableFuncExpr>
optFuncExprs, AbstractLogicalOperator op,
+ Dataset dataset, IOptimizationContext context, IAType filterType)
throws AlgebricksException {
+ ArrayList<UnnestMapOperator> primaryUnnestMapOps = new ArrayList<>();
+ boolean hasSecondaryIndexMap = false;
Queue<Mutable<ILogicalOperator>> queue = new
LinkedList<>(op.getInputs());
while (!queue.isEmpty()) {
AbstractLogicalOperator descendantOp = (AbstractLogicalOperator)
queue.poll().getValue();
@@ -195,8 +207,8 @@
List<LogicalVariable> minFilterVars = new
ArrayList<>();
List<LogicalVariable> maxFilterVars = new
ArrayList<>();
- AssignOperator assignOp =
createAssignOperator(optFuncExprs, minFilterVars, maxFilterVars,
- context);
+ AssignOperator assignOp =
+ createAssignOperator(optFuncExprs,
minFilterVars, maxFilterVars, context);
unnestMapOp.setMinFilterVars(minFilterVars);
unnestMapOp.setMaxFilterVars(maxFilterVars);
@@ -210,10 +222,135 @@
assignOp.getInputs()
.add(new
MutableObject<ILogicalOperator>(unnestMapOp.getInputs().get(0).getValue()));
unnestMapOp.getInputs().get(0).setValue(assignOp);
+
+ if (jobGenParams.isPrimaryIndex) {
+ primaryUnnestMapOps.add(unnestMapOp);
+ } else {
+ hasSecondaryIndexMap = true;
+ }
}
}
}
queue.addAll(descendantOp.getInputs());
+ }
+ if (hasSecondaryIndexMap && !primaryUnnestMapOps.isEmpty()) {
+ propagateFilterToPrimaryIndex(primaryUnnestMapOps, filterType,
context);
+ }
+ }
+
+ private void propagateFilterToPrimaryIndex(ArrayList<UnnestMapOperator>
primaryUnnestMapOps, IAType filterType,
+ IOptimizationContext context) throws AlgebricksException {
+ for (UnnestMapOperator primaryOp : primaryUnnestMapOps) {
+ Mutable<ILogicalOperator> assignOrOrderOrIntersect =
primaryOp.getInputs().get(0);
+ Mutable<ILogicalOperator> intersectOrSort = null;
+ switch (assignOrOrderOrIntersect.getValue().getOperatorTag()) {
+ case ASSIGN: // already has the filter condition value from
the query
+ intersectOrSort =
assignOrOrderOrIntersect.getValue().getInputs().get(0);
+ break;
+ case INTERSECT:
+ case ORDER:
+ intersectOrSort = assignOrOrderOrIntersect;
+ break;
+ default:
+ LOGGER.warning("Unexpected operator:" +
assignOrOrderOrIntersect.getValue().getOperatorTag()
+ + " between secondary to primary index access
path, expected ASSIGN, ORDER, or INTERSECT");
+ return;
+ }
+
+ boolean indeedSecondaryToPrimaryPath = false;
+
+ switch (intersectOrSort.getValue().getOperatorTag()) {
+ case INTERSECT:
+ for (Mutable<ILogicalOperator> mutableOp :
intersectOrSort.getValue().getInputs()) {
+ ILogicalOperator child = mutableOp.getValue();
+ if
(child.getOperatorTag().equals(LogicalOperatorTag.UNNEST_MAP)) {
+ indeedSecondaryToPrimaryPath = true;
+
propagateFilterInSecondaryUnnest((UnnestMapOperator) child, filterType,
context);
+ }
+ }
+ break;
+ case ORDER:
+ ILogicalOperator child =
intersectOrSort.getValue().getInputs().get(0).getValue();
+ if
(child.getOperatorTag().equals(LogicalOperatorTag.UNNEST_MAP)) {
+ indeedSecondaryToPrimaryPath = true;
+ propagateFilterInSecondaryUnnest((UnnestMapOperator)
child, filterType, context);
+ }
+ break;
+ default:
+ LOGGER.warning("Unexpected operator:" +
intersectOrSort.getValue().getOperatorTag()
+ + " between secondary to primary index access
path, expected ORDER or INTERSECT");
+ break;
+ }
+
+ if (indeedSecondaryToPrimaryPath) {
+ setFilterValueFromSecondaryUnnest(primaryOp, intersectOrSort,
context);
+ }
+ }
+ }
+
+ private void propagateFilterInSecondaryUnnest(UnnestMapOperator
secondaryUnnest, IAType filterType,
+ IOptimizationContext context) throws AlgebricksException {
+
+ LogicalVariable minIndexFilterVar = context.newVar();
+ LogicalVariable maxIndexFilterVar = context.newVar();
+ secondaryUnnest.setPropagateIndexFilter(true);
+ secondaryUnnest.getVariables().add(minIndexFilterVar);
+ secondaryUnnest.getVariableTypes().add(filterType);
+ secondaryUnnest.getVariables().add(maxIndexFilterVar);
+ secondaryUnnest.getVariableTypes().add(filterType);
+
+ context.computeAndSetTypeEnvironmentForOperator(secondaryUnnest);
+ }
+
+ private void setFilterValueFromSecondaryUnnest(UnnestMapOperator primaryOp,
+ Mutable<ILogicalOperator> intersectOrSort, IOptimizationContext
context) throws AlgebricksException {
+ UnnestMapOperator secondaryMap = (UnnestMapOperator)
intersectOrSort.getValue().getInputs().get(0).getValue();
+
primaryOp.setMinFilterVars(Collections.singletonList(secondaryMap.getPropagateIndexMinFilterVar()));
+
primaryOp.setMaxFilterVars(Collections.singletonList(secondaryMap.getPropagateIndexMaxFilterVar()));
+
+ List<Mutable<ILogicalExpression>> indexFilterExpression =
Arrays.asList(
+ (new MutableObject<>(new
VariableReferenceExpression(secondaryMap.getPropagateIndexMinFilterVar()))),
+ (new MutableObject<>(new
VariableReferenceExpression(secondaryMap.getPropagateIndexMaxFilterVar()))));
+
+ primaryOp.setAdditionalFilteringExpressions(indexFilterExpression);
+ primaryOp.getInputs().set(0, intersectOrSort);
+ context.computeAndSetTypeEnvironmentForOperator(primaryOp);
+ }
+
+ private void assignFilterFromSecondaryUnnest(AbstractLogicalOperator op,
Dataset dataset,
+ IOptimizationContext context, IAType filterType) throws
AlgebricksException {
+ ArrayList<UnnestMapOperator> primaryUnnestMapOps = new ArrayList<>();
+ boolean hasSecondaryIndexMap = false;
+ Queue<Mutable<ILogicalOperator>> queue = new
LinkedList<>(op.getInputs());
+ while (!queue.isEmpty()) {
+ AbstractLogicalOperator descendantOp = (AbstractLogicalOperator)
queue.poll().getValue();
+ if (descendantOp == null) {
+ continue;
+ }
+ if (descendantOp.getOperatorTag() ==
LogicalOperatorTag.UNNEST_MAP) {
+ UnnestMapOperator unnestMapOp = (UnnestMapOperator)
descendantOp;
+ ILogicalExpression unnestExpr =
unnestMapOp.getExpressionRef().getValue();
+ if (unnestExpr.getExpressionTag() ==
LogicalExpressionTag.FUNCTION_CALL) {
+ AbstractFunctionCallExpression f =
(AbstractFunctionCallExpression) unnestExpr;
+ FunctionIdentifier fid = f.getFunctionIdentifier();
+ if (!fid.equals(BuiltinFunctions.INDEX_SEARCH)) {
+ throw new IllegalStateException();
+ }
+ AccessMethodJobGenParams jobGenParams = new
AccessMethodJobGenParams();
+ jobGenParams.readFromFuncArgs(f.getArguments());
+ if
(dataset.getDatasetName().compareTo(jobGenParams.datasetName) == 0) {
+ if (jobGenParams.isPrimaryIndex) {
+ primaryUnnestMapOps.add(unnestMapOp);
+ } else {
+ hasSecondaryIndexMap = true;
+ }
+ }
+ }
+ }
+ queue.addAll(descendantOp.getInputs());
+ }
+ if (hasSecondaryIndexMap && !primaryUnnestMapOps.isEmpty()) {
+ propagateFilterToPrimaryIndex(primaryUnnestMapOps, filterType,
context);
}
}
@@ -246,8 +383,7 @@
} else {
throw new AlgebricksException("Unexpected function for
Unnest Map: " + fid);
}
- return ((MetadataProvider)
context.getMetadataProvider()).findDataset(dataverseName,
- datasetName);
+ return ((MetadataProvider)
context.getMetadataProvider()).findDataset(dataverseName, datasetName);
}
}
if (descendantOp.getInputs().isEmpty()) {
@@ -318,8 +454,8 @@
if (funcVarIndex == -1) {
continue;
}
- List<String> fieldName =
getFieldNameFromSubAssignTree(optFuncExpr, descendantOp, varIndex,
- recType).second;
+ List<String> fieldName =
+ getFieldNameFromSubAssignTree(optFuncExpr,
descendantOp, varIndex, recType).second;
if (fieldName == null) {
return false;
}
@@ -411,8 +547,7 @@
}
AbstractFunctionCallExpression funcExpr =
(AbstractFunctionCallExpression) expr;
FunctionIdentifier funcIdent = funcExpr.getFunctionIdentifier();
- if (funcIdent == BuiltinFunctions.FIELD_ACCESS_BY_NAME
- || funcIdent == BuiltinFunctions.FIELD_ACCESS_BY_INDEX) {
+ if (funcIdent == BuiltinFunctions.FIELD_ACCESS_BY_NAME || funcIdent ==
BuiltinFunctions.FIELD_ACCESS_BY_INDEX) {
//get the variable from here. Figure out which input it came from.
Go to that input!!!
ArrayList<LogicalVariable> usedVars = new ArrayList<>();
diff --git
a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter/btree-btree-search-wo-query-filter.aql
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter/btree-btree-search-wo-query-filter.aql
new file mode 100644
index 0000000..f5500d9
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter/btree-btree-search-wo-query-filter.aql
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type tTweet as closed {
+ id: int32,
+ location: point,
+ message: string,
+ create_at: datetime,
+ misc: string
+}
+
+create dataset dsTweet(tTweet) primary key id with filter on create_at;
+
+create index misc_index on dsTweet(misc) type btree;
+
+write output to nc1:"rttest/filter/btree-btree-search-wo-query-filter.adm";
+
+for $t in dataset dsTweet
+where $t.misc = "xyz"
+return $t
diff --git
a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter/btree-btree-search.aql
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter/btree-btree-search.aql
new file mode 100644
index 0000000..2d8253e
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter/btree-btree-search.aql
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type tTweet as closed {
+ id: int32,
+ location: point,
+ message: string,
+ create_at: datetime,
+ misc: string
+}
+
+create dataset dsTweet(tTweet) primary key id with filter on create_at;
+
+create index misc_index on dsTweet(misc) type btree;
+
+write output to nc1:"rttest/filter/btree-btree-search.adm";
+
+let $ts_start := datetime("2015-11-11T00:00:00Z")
+let $ts_end := datetime("2015-12-18T23:59:59Z")
+
+for $t in dataset dsTweet
+where $t.create_at >= $ts_start and $t.create_at < $ts_end
+ and $t.misc = "xyz"
+return $t
diff --git
a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter/inverted-btree-search-wo-query-filter.aql
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter/inverted-btree-search-wo-query-filter.aql
new file mode 100644
index 0000000..3286078
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter/inverted-btree-search-wo-query-filter.aql
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type tTweet as closed {
+ id: int32,
+ location: point,
+ message: string,
+ create_at: datetime,
+ misc: string
+}
+
+create dataset dsTweet(tTweet) primary key id with filter on create_at;
+
+create index ft_index on dsTweet(message) type fulltext;
+
+write output to nc1:"rttest/filter/inverted-btree-search-wo-query-filter.adm";
+
+let $keyword := "hello"
+
+for $t in dataset dsTweet
+where ftcontains($t.message, $keyword)
+return $t
diff --git
a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter/inverted-btree-search.aql
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter/inverted-btree-search.aql
new file mode 100644
index 0000000..c7ebfb1
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter/inverted-btree-search.aql
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type tTweet as closed {
+ id: int32,
+ location: point,
+ message: string,
+ create_at: datetime,
+ misc: string
+}
+
+create dataset dsTweet(tTweet) primary key id with filter on create_at;
+
+create index ft_index on dsTweet(message) type fulltext;
+
+write output to nc1:"rttest/filter/inverted-btree-search.adm";
+
+let $ts_start := datetime("2015-11-11T00:00:00Z")
+let $ts_end := datetime("2015-12-18T23:59:59Z")
+let $keyword := "hello"
+
+for $t in dataset dsTweet
+where $t.create_at >= $ts_start and $t.create_at < $ts_end
+ and ftcontains($t.message, $keyword)
+return $t
diff --git
a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter/multi-index-btree-search-wo-query-filter.aql
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter/multi-index-btree-search-wo-query-filter.aql
new file mode 100644
index 0000000..7864d9a
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter/multi-index-btree-search-wo-query-filter.aql
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type tTweet as closed {
+ id: int32,
+ location: point,
+ message: string,
+ create_at: datetime,
+ misc: string
+}
+
+create dataset dsTweet(tTweet) primary key id with filter on create_at;
+
+create index ft_index on dsTweet(message) type fulltext;
+create index misc_index on dsTweet(misc) type btree;
+create index location_index on dsTweet(location) type rtree;
+
+write output to
nc1:"rttest/filter/multi-index-btree-search-wo-query-filter.adm";
+
+let $region := create-rectangle(create-point(-128.43,20.29),
create-point(-64.26,54.56))
+let $keyword := "hello"
+
+for $t in dataset dsTweet
+where $t.misc < "xzy"
+ and spatial-intersect($t.location, $region)
+ and ftcontains($t.message, $keyword)
+return $t
diff --git
a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter/multi-index-btree-search.aql
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter/multi-index-btree-search.aql
new file mode 100644
index 0000000..8ebb597
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter/multi-index-btree-search.aql
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type tTweet as closed {
+ id: int32,
+ location: point,
+ message: string,
+ create_at: datetime,
+ misc: string
+}
+
+create dataset dsTweet(tTweet) primary key id with filter on create_at;
+
+create index ft_index on dsTweet(message) type fulltext;
+create index misc_index on dsTweet(misc) type btree;
+create index location_index on dsTweet(location) type rtree;
+
+write output to nc1:"rttest/filter/multi-index-btree-search.adm";
+
+let $region := create-rectangle(create-point(-128.43,20.29),
create-point(-64.26,54.56))
+let $ts_start := datetime("2015-11-11T00:00:00Z")
+let $ts_end := datetime("2015-12-18T23:59:59Z")
+let $keyword := "hello"
+
+for $t in dataset dsTweet
+where $t.create_at >= $ts_start and $t.create_at < $ts_end
+ and $t.misc < "xzy"
+ and spatial-intersect($t.location, $region)
+ and ftcontains($t.message, $keyword)
+return $t
diff --git
a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter/rtree-btree-search-wo-query-filter.aql
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter/rtree-btree-search-wo-query-filter.aql
new file mode 100644
index 0000000..e36290e
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter/rtree-btree-search-wo-query-filter.aql
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type tTweet as closed {
+ id: int32,
+ location: point,
+ message: string,
+ create_at: datetime,
+ misc: string
+}
+
+create dataset dsTweet(tTweet) primary key id with filter on create_at;
+
+create index location_index on dsTweet(location) type rtree;
+
+write output to nc1:"rttest/filter/rtree-btree-search-wo-query-filter.adm";
+
+let $region := create-rectangle(create-point(-128.43,20.29),
create-point(-64.26,54.56))
+
+for $t in dataset dsTweet
+where spatial-intersect($t.location, $region)
+return $t
diff --git
a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter/rtree-btree-search.aql
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter/rtree-btree-search.aql
new file mode 100644
index 0000000..909e36e
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter/rtree-btree-search.aql
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type tTweet as closed {
+ id: int32,
+ location: point,
+ message: string,
+ create_at: datetime,
+ misc: string
+}
+
+create dataset dsTweet(tTweet) primary key id with filter on create_at;
+
+create index location_index on dsTweet(location) type rtree;
+
+write output to nc1:"rttest/filter/rtree-btree-search.adm";
+
+let $region := create-rectangle(create-point(-128.43,20.29),
create-point(-64.26,54.56))
+let $ts_start := datetime("2015-11-11T00:00:00Z")
+let $ts_end := datetime("2015-12-18T23:59:59Z")
+
+for $t in dataset dsTweet
+where $t.create_at >= $ts_start and $t.create_at < $ts_end
+ and spatial-intersect($t.location, $region)
+return $t
diff --git
a/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter/btree-btree-search-wo-query-filter.plan
b/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter/btree-btree-search-wo-query-filter.plan
new file mode 100644
index 0000000..d7667d6
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter/btree-btree-search-wo-query-filter.plan
@@ -0,0 +1,15 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$9(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git
a/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter/btree-btree-search.plan
b/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter/btree-btree-search.plan
new file mode 100644
index 0000000..0d9a257
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter/btree-btree-search.plan
@@ -0,0 +1,17 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$19(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git
a/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter/inverted-btree-search-wo-query-filter.plan
b/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter/inverted-btree-search-wo-query-filter.plan
new file mode 100644
index 0000000..1b99cd6
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter/inverted-btree-search-wo-query-filter.plan
@@ -0,0 +1,13 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$8(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- SINGLE_PARTITION_INVERTED_INDEX_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git
a/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter/inverted-btree-search.plan
b/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter/inverted-btree-search.plan
new file mode 100644
index 0000000..7b03ff5
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter/inverted-btree-search.plan
@@ -0,0 +1,15 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$18(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- SINGLE_PARTITION_INVERTED_INDEX_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git
a/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter/multi-index-btree-search-wo-query-filter.plan
b/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter/multi-index-btree-search-wo-query-filter.plan
new file mode 100644
index 0000000..8c799fa
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter/multi-index-btree-search-wo-query-filter.plan
@@ -0,0 +1,33 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- INTERSECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$22(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$24(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- SINGLE_PARTITION_INVERTED_INDEX_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$33(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- RTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git
a/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter/multi-index-btree-search.plan
b/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter/multi-index-btree-search.plan
new file mode 100644
index 0000000..b7e3da8
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter/multi-index-btree-search.plan
@@ -0,0 +1,37 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- INTERSECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$31(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$33(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- SINGLE_PARTITION_INVERTED_INDEX_SEARCH
|PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$42(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- RTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git
a/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter/rtree-btree-search-wo-query-filter.plan
b/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter/rtree-btree-search-wo-query-filter.plan
new file mode 100644
index 0000000..59443b5
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter/rtree-btree-search-wo-query-filter.plan
@@ -0,0 +1,15 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$20(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- RTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git
a/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter/rtree-btree-search.plan
b/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter/rtree-btree-search.plan
new file mode 100644
index 0000000..8f346d3
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter/rtree-btree-search.plan
@@ -0,0 +1,17 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$30(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- RTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
index 68c7e22..e2b1761 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
@@ -120,7 +120,7 @@
int[] maxFilterFieldIndexes =
createFilterIndexes(maxFilterVars, opSchema);
return metadataProvider.buildBtreeRuntime(jobSpec, opSchema,
typeEnv, context, true,
false, ((DatasetDataSource) dataSource).getDataset(),
primaryIndex.getIndexName(), null, null,
- true, true, minFilterFieldIndexes,
maxFilterFieldIndexes);
+ true, true, false, minFilterFieldIndexes,
maxFilterFieldIndexes);
default:
throw new AlgebricksException("Unknown datasource type");
}
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index d111eb2..498c717 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -447,8 +447,8 @@
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>
buildBtreeRuntime(JobSpecification jobSpec,
IOperatorSchema opSchema, IVariableTypeEnvironment typeEnv,
JobGenContext context, boolean retainInput,
boolean retainMissing, Dataset dataset, String indexName, int[]
lowKeyFields, int[] highKeyFields,
- boolean lowKeyInclusive, boolean highKeyInclusive, int[]
minFilterFieldIndexes, int[] maxFilterFieldIndexes)
- throws AlgebricksException {
+ boolean lowKeyInclusive, boolean highKeyInclusive, boolean
propagateFilter, int[] minFilterFieldIndexes,
+ int[] maxFilterFieldIndexes) throws AlgebricksException {
boolean isSecondary = true;
int numSecondaryKeys = 0;
try {
@@ -533,7 +533,7 @@
lowKeyInclusive, highKeyInclusive,
dataset.getIndexDataflowHelperFactory(this, theIndex,
itemType, metaType, compactionInfo.first,
compactionInfo.second),
- retainInput, retainMissing,
context.getMissingWriterFactory(), searchCallbackFactory,
+ retainInput, retainMissing,
context.getMissingWriterFactory(), searchCallbackFactory, propagateFilter,
minFilterFieldIndexes, maxFilterFieldIndexes,
metadataPageManagerFactory);
} else {
IIndexDataflowHelperFactory indexDataflowHelperFactory =
dataset.getIndexDataflowHelperFactory(this,
@@ -553,7 +553,8 @@
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>
buildRtreeRuntime(JobSpecification jobSpec,
List<LogicalVariable> outputVars, IOperatorSchema opSchema,
IVariableTypeEnvironment typeEnv,
JobGenContext context, boolean retainInput, boolean retainMissing,
Dataset dataset, String indexName,
- int[] keyFields, int[] minFilterFieldIndexes, int[]
maxFilterFieldIndexes) throws AlgebricksException {
+ int[] keyFields, boolean propagateFilter, int[]
minFilterFieldIndexes, int[] maxFilterFieldIndexes)
+ throws AlgebricksException {
try {
ARecordType recType = (ARecordType)
findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName());
int numPrimaryKeys =
DatasetUtil.getPartitioningKeys(dataset).size();
@@ -638,7 +639,7 @@
rtreeSearchOp = new RTreeSearchOperatorDescriptor(jobSpec,
outputRecDesc,
appContext.getStorageManager(),
appContext.getIndexLifecycleManagerProvider(), spPc.first,
typeTraits, comparatorFactories, keyFields,
indexDataflowHelperFactory, retainInput,
- retainMissing, context.getMissingWriterFactory(),
searchCallbackFactory, minFilterFieldIndexes,
+ retainMissing, context.getMissingWriterFactory(),
searchCallbackFactory, propagateFilter, minFilterFieldIndexes,
maxFilterFieldIndexes, metadataPageManagerFactory);
} else {
// Create the operator
diff --git
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractScanOperator.java
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractScanOperator.java
index 9cd7138..5d6f40c 100644
---
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractScanOperator.java
+++
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractScanOperator.java
@@ -36,6 +36,11 @@
return variables;
}
+ public List<LogicalVariable> getScanVariables() {
+ return variables;
+ }
+
+
public void setVariables(List<LogicalVariable> variables) {
this.variables = variables;
}
diff --git
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractUnnestMapOperator.java
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractUnnestMapOperator.java
index 8a2981d..5ed7103 100644
---
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractUnnestMapOperator.java
+++
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractUnnestMapOperator.java
@@ -35,12 +35,24 @@
protected List<LogicalVariable> minFilterVars;
protected List<LogicalVariable> maxFilterVars;
+ protected boolean propagateIndexFilter;
+
public AbstractUnnestMapOperator(List<LogicalVariable> variables,
Mutable<ILogicalExpression> expression,
List<Object> variableTypes, boolean propagateInput) {
super(variables, expression);
this.expression = expression;
this.variableTypes = variableTypes;
this.propagateInput = propagateInput;
+ this.propagateIndexFilter = false;
+ }
+
+ @Override
+ public List<LogicalVariable> getScanVariables() {
+ if (propagateIndexFilter) {
+ return variables.subList(0, variables.size() - 2);
+ } else {
+ return variables;
+ }
}
public List<Object> getVariableTypes() {
@@ -98,4 +110,27 @@
return additionalFilteringExpressions;
}
+ public void setPropagateIndexFilter(boolean propagateIndexFilter) {
+ this.propagateIndexFilter = propagateIndexFilter;
+ }
+
+ public boolean getPropagateIndexFilter() {
+ return this.propagateIndexFilter;
+ }
+
+ public LogicalVariable getPropagateIndexMinFilterVar() {
+ if (propagateIndexFilter) {
+ return variables.get(variables.size() - 2);
+ } else {
+ return null;
+ }
+ }
+
+ public LogicalVariable getPropagateIndexMaxFilterVar() {
+ if (propagateIndexFilter) {
+ return variables.get(variables.size() - 1);
+ } else {
+ return null;
+ }
+ }
}
diff --git
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java
index 89e2423..e7fb6c0 100644
---
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java
+++
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java
@@ -51,7 +51,7 @@
// this operator propagates all input variables.
@Override
public IVariableTypeEnvironment
computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
- IVariableTypeEnvironment env = null;
+ IVariableTypeEnvironment env;
if (propagateInput) {
env = createPropagatingAllInputsTypeEnvironment(ctx);
} else {
diff --git
a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/AbstractRTreeOperatorTest.java
b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/AbstractRTreeOperatorTest.java
index 0d4bf92..03647e7 100644
---
a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/AbstractRTreeOperatorTest.java
+++
b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/AbstractRTreeOperatorTest.java
@@ -20,11 +20,7 @@
package org.apache.hyracks.tests.am.rtree;
import org.apache.hyracks.api.constraints.PartitionConstraintHelper;
-import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import org.apache.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
-import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
-import org.apache.hyracks.api.dataflow.value.ITypeTraits;
-import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.dataflow.value.*;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.io.ManagedFileSplit;
--
To view, visit https://asterix-gerrit.ics.uci.edu/1727
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I0e2fe0208662e5dcd49d1a22bfb58f96533e9497
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Jianfeng Jia <[email protected]>
