Luo Chen has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/2541
Change subject: [ASTERIXDB-2344] SELECT/LIMIT pushdown for primary index
......................................................................
[ASTERIXDB-2344] SELECT/LIMIT pushdown for primary index
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
-Implement SELECT pushdown for primary index scan/lookup
-Implement LIMIT pushdown for primary index scan/lookup
Change-Id: I824fcad79995325e12a1a81d629160025294b915
---
M
asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
M
asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
A
asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushLimitIntoPrimaryRule.java
A
asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushSelectIntoPrimaryRule.java
A
asterixdb/asterix-app/src/test/resources/optimizerts/queries/push-limit-to-lookup.sqlpp
A
asterixdb/asterix-app/src/test/resources/optimizerts/queries/push-limit-to-scan.sqlpp
A
asterixdb/asterix-app/src/test/resources/optimizerts/queries/push-select-to-lookup.sqlpp
A
asterixdb/asterix-app/src/test/resources/optimizerts/queries/push-select-to-scan.sqlpp
A
asterixdb/asterix-app/src/test/resources/optimizerts/results/push-limit-to-lookup.plan
A
asterixdb/asterix-app/src/test/resources/optimizerts/results/push-limit-to-scan.plan
A
asterixdb/asterix-app/src/test/resources/optimizerts/results/push-select-to-lookup.plan
A
asterixdb/asterix-app/src/test/resources/optimizerts/results/push-select-to-scan.plan
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-scan-select/limit-scan-select.1.ddl.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-scan-select/limit-scan-select.2.update.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-scan-select/limit-scan-select.3.query.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-scan-select/limit-scan-select.4.query.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-scan/limit-scan.1.ddl.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-scan/limit-scan.2.update.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-scan/limit-scan.3.query.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-scan/limit-scan.4.query.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-secondary-index/limit-secondary-index.1.ddl.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-secondary-index/limit-secondary-index.2.update.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-secondary-index/limit-secondary-index.3.query.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-secondary-index/limit-secondary-index.4.query.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/results/limit/limit-scan-select/limit-scan-select.3.adm
A
asterixdb/asterix-app/src/test/resources/runtimets/results/limit/limit-scan-select/limit-scan-select.4.adm
A
asterixdb/asterix-app/src/test/resources/runtimets/results/limit/limit-scan/limit-scan.3.adm
A
asterixdb/asterix-app/src/test/resources/runtimets/results/limit/limit-scan/limit-scan.4.adm
A
asterixdb/asterix-app/src/test/resources/runtimets/results/limit/limit-secondary-index/limit-secondary-index.3.adm
A
asterixdb/asterix-app/src/test/resources/runtimets/results/limit/limit-secondary-index/limit-secondary-index.4.adm
M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
M
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DataSource.java
M
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
M
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
M
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
M
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java
M
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
M hyracks-fullstack/algebricks/algebricks-core/pom.xml
M
hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
M
hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java
M
hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java
M
hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java
M
hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java
M
hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorDescriptor.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorNodePushable.java
48 files changed, 1,150 insertions(+), 45 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/41/2541/1
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
index beb47a8..28369ef 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
@@ -47,6 +47,7 @@
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestMapOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
+import
org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
import
org.apache.hyracks.algebricks.core.algebra.properties.BroadcastPartitioningProperty;
import
org.apache.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
import org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain;
@@ -59,6 +60,7 @@
import
org.apache.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
+import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
/**
* Contributes the runtime operator for an unnest-map representing a BTree
search.
@@ -120,13 +122,23 @@
MetadataProvider metadataProvider = (MetadataProvider)
context.getMetadataProvider();
Dataset dataset =
metadataProvider.findDataset(jobGenParams.getDataverseName(),
jobGenParams.getDatasetName());
IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(op);
+ ITupleFilterFactory tupleFilterFactory = null;
+ long outputLimit = -1;
+ if (unnestMap instanceof UnnestMapOperator) {
+ UnnestMapOperator unnestMapOp = (UnnestMapOperator) unnestMap;
+ outputLimit = unnestMapOp.getOutputLimit();
+ if (unnestMapOp.getSelectCondition() != null) {
+ tupleFilterFactory =
metadataProvider.getTupleFilterFactory(new IOperatorSchema[] { opSchema },
typeEnv,
+ unnestMapOp.getSelectCondition().getValue(), context);
+ }
+ }
// By nature, LEFT_OUTER_UNNEST_MAP should generate null values for
non-matching tuples.
boolean retainMissing = op.getOperatorTag() ==
LogicalOperatorTag.LEFT_OUTER_UNNEST_MAP;
Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> btreeSearch =
metadataProvider.buildBtreeRuntime(
builder.getJobSpec(), opSchema, typeEnv, context,
jobGenParams.getRetainInput(), retainMissing, dataset,
jobGenParams.getIndexName(), lowKeyIndexes, highKeyIndexes,
jobGenParams.isLowKeyInclusive(),
jobGenParams.isHighKeyInclusive(), propagateFilter,
minFilterFieldIndexes, maxFilterFieldIndexes,
- unnestMap.getGenerateCallBackProceedResultVar());
+ tupleFilterFactory, outputLimit,
unnestMap.getGenerateCallBackProceedResultVar());
builder.contributeHyracksOperator(unnestMap, btreeSearch.first);
builder.contributeAlgebricksPartitionConstraint(btreeSearch.first,
btreeSearch.second);
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
index cd217ab..82c3d33 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
@@ -63,7 +63,9 @@
import org.apache.asterix.optimizer.rules.PushFieldAccessRule;
import org.apache.asterix.optimizer.rules.PushGroupByThroughProduct;
import org.apache.asterix.optimizer.rules.PushLimitIntoOrderByRule;
+import org.apache.asterix.optimizer.rules.PushLimitIntoPrimaryRule;
import org.apache.asterix.optimizer.rules.PushProperJoinThroughProduct;
+import org.apache.asterix.optimizer.rules.PushSelectIntoPrimaryRule;
import org.apache.asterix.optimizer.rules.PushSimilarityFunctionsBelowJoin;
import
org.apache.asterix.optimizer.rules.RemoveLeftOuterUnnestForLeftOuterJoinRule;
import org.apache.asterix.optimizer.rules.RemoveRedundantListifyRule;
@@ -350,6 +352,8 @@
// We are going to apply a constant folding rule again for this case.
physicalRewritesTopLevel.add(new ConstantFoldingRule(appCtx));
physicalRewritesTopLevel.add(new PushLimitIntoOrderByRule());
+ physicalRewritesTopLevel.add(new PushSelectIntoPrimaryRule());
+ physicalRewritesTopLevel.add(new PushLimitIntoPrimaryRule());
physicalRewritesTopLevel.add(new IntroduceProjectsRule());
physicalRewritesTopLevel.add(new SetAlgebricksPhysicalOperatorsRule());
physicalRewritesTopLevel.add(new
IntroduceRapidFrameFlushProjectAssignRule());
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushLimitIntoPrimaryRule.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushLimitIntoPrimaryRule.java
new file mode 100644
index 0000000..83fd0fe
--- /dev/null
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushLimitIntoPrimaryRule.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.optimizer.rules;
+
+import org.apache.asterix.optimizer.rules.am.AccessMethodUtils;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import
org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import
org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import
org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * If primary unnest map or dataset scan is followed by a limit operator, then
+ * we push the limit into the primary operator
+ *
+ */
+public class PushLimitIntoPrimaryRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef,
IOptimizationContext context) {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef,
IOptimizationContext context)
+ throws AlgebricksException {
+ ILogicalOperator input = opRef.getValue();
+ // The current operator should be LIMIT operator.
+ if (input.getOperatorTag() != LogicalOperatorTag.LIMIT) {
+ return false;
+ }
+ if (context.checkIfInDontApplySet(this, input)) {
+ return false;
+ }
+ context.addToDontApplySet(this, input);
+
+ ILogicalOperator childOp = input.getInputs().get(0).getValue();
+ if (childOp.getOperatorTag() == LogicalOperatorTag.EXCHANGE) {
+ // bypass EXCHANGE
+ childOp = childOp.getInputs().get(0).getValue();
+ }
+
+ long outputLimit = getOutputLimit((LimitOperator) input);
+ if (childOp.getOperatorTag() == LogicalOperatorTag.SELECT) {
+ return
setLimitForScanOrUnnestMap(childOp.getInputs().get(0).getValue(), outputLimit);
+ } else {
+ return setLimitForScanOrUnnestMap(childOp, outputLimit);
+ }
+ }
+
+ private long getOutputLimit(LimitOperator limit) {
+ long outputLimit =
AccessMethodUtils.getInt64Constant(limit.getMaxObjects());
+ if (limit.getOffset() != null && limit.getOffset().getValue() != null)
{
+ outputLimit +=
AccessMethodUtils.getInt64Constant(limit.getOffset());
+ }
+ return outputLimit;
+ }
+
+ private boolean setLimitForScanOrUnnestMap(ILogicalOperator op, long
outputLimit) throws AlgebricksException {
+ if (op.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
+ ((DataSourceScanOperator) op).setOutputLimit(outputLimit);
+ return true;
+ } else if (op.getOperatorTag() == LogicalOperatorTag.UNNEST_MAP) {
+ ((UnnestMapOperator) op).setOutputLimit(outputLimit);
+ return true;
+ } else {
+ return false;
+ }
+ }
+}
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushSelectIntoPrimaryRule.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushSelectIntoPrimaryRule.java
new file mode 100644
index 0000000..e0044de
--- /dev/null
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushSelectIntoPrimaryRule.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.optimizer.rules;
+
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.metadata.declared.DatasetDataSource;
+import org.apache.asterix.optimizer.rules.am.AccessMethodJobGenParams;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import
org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import
org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import
org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import
org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * This rule performs two operators:
+ * Push SELECT operator into a dataset scan or primary UNNEST_MAP
+ * PATTERN: SCAN/UNNEST_MAP -> SELECT
+ */
+public class PushSelectIntoPrimaryRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef,
IOptimizationContext context) {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef,
IOptimizationContext context)
+ throws AlgebricksException {
+ ILogicalOperator op = opRef.getValue();
+ if (context.checkIfInDontApplySet(this, op)) {
+ return false;
+ }
+ context.addToDontApplySet(this, op);
+ if (op.getInputs().isEmpty()) {
+ return false;
+ }
+
+ boolean changed = false;
+ for (Mutable<ILogicalOperator> input : op.getInputs()) {
+ changed = rewriteSelect(input, context) || changed;
+ }
+ if (changed) {
+ context.computeAndSetTypeEnvironmentForOperator(op);
+ }
+ return changed;
+ }
+
+ private boolean rewriteSelect(Mutable<ILogicalOperator> input,
IOptimizationContext ctx)
+ throws AlgebricksException {
+ if (input.getValue().getOperatorTag() != LogicalOperatorTag.SELECT) {
+ return false;
+ }
+
+ SelectOperator select = (SelectOperator) input.getValue();
+
+ ILogicalOperator child = select.getInputs().get(0).getValue();
+ LogicalOperatorTag childTag = child.getOperatorTag();
+ if (childTag == LogicalOperatorTag.DATASOURCESCAN) {
+ return rewriteSelectForScan(select, (DataSourceScanOperator)
child, ctx);
+ } else if (childTag == LogicalOperatorTag.UNNEST_MAP) {
+ return rewriteSelectForUnnestMap(select, (UnnestMapOperator)
child, ctx);
+ }
+
+ return false;
+ }
+
+ private boolean rewriteSelectForScan(SelectOperator select,
DataSourceScanOperator scan, IOptimizationContext ctx)
+ throws AlgebricksException {
+ if (scan.getDataSource() instanceof DatasetDataSource) {
+ if (select.getRetainMissing()) {
+ throw
CompilationException.create(ErrorCode.COMPILATION_ILLEGAL_STATE,
+ select.getOperatorTag().toString());
+ }
+ scan.setSelectCondition(new
MutableObject<>(select.getCondition().getValue()));
+ // TODO remove SELECT operator for INTERNAL dataset
+ return true;
+ }
+ return false;
+ }
+
+ private boolean rewriteSelectForUnnestMap(SelectOperator select,
UnnestMapOperator unnestMap,
+ IOptimizationContext ctx) throws AlgebricksException {
+ ILogicalExpression unnestExpr =
unnestMap.getExpressionRef().getValue();
+ if (unnestExpr.getExpressionTag() ==
LogicalExpressionTag.FUNCTION_CALL) {
+ AbstractFunctionCallExpression f =
(AbstractFunctionCallExpression) unnestExpr;
+ AccessMethodJobGenParams jobGenParams = new
AccessMethodJobGenParams();
+ jobGenParams.readFromFuncArgs(f.getArguments());
+ if (jobGenParams.isPrimaryIndex()) {
+ if (select.getRetainMissing()) {
+ throw
CompilationException.create(ErrorCode.COMPILATION_ILLEGAL_STATE,
+ select.getOperatorTag().toString());
+ }
+ unnestMap.setSelectCondition(new
MutableObject<>(select.getCondition().getValue()));
+ // technically, we can remove select operator.
+ // however, this will change lots of existing test cases.
+ // TODO: remove SELECT operator in the future
+ return true;
+ }
+ }
+ return false;
+ }
+
+}
diff --git
a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/push-limit-to-lookup.sqlpp
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/push-limit-to-lookup.sqlpp
new file mode 100644
index 0000000..60fede3
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/push-limit-to-lookup.sqlpp
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+drop dataverse tpch if exists;
+create dataverse tpch;
+use tpch;
+
+create type OrderType as closed {
+ o_orderkey: int32,
+ o_custkey: int32,
+ o_orderstatus: string,
+ o_totalprice: double,
+ o_orderdate: string,
+ o_orderpriority: string,
+ o_clerk: string,
+ o_shippriority: int32,
+ o_comment: string
+};
+
+create dataset Orders(OrderType) primary key o_orderkey;
+
+create index statusIndex on Orders(o_orderstatus) type btree;
+
+select *
+from Orders
+where o_totalprice > 100 AND o_orderstatus = "finished"
+limit 10;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/push-limit-to-scan.sqlpp
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/push-limit-to-scan.sqlpp
new file mode 100644
index 0000000..cee0a38
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/push-limit-to-scan.sqlpp
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+drop dataverse tpch if exists;
+create dataverse tpch;
+use tpch;
+
+create type OrderType as closed {
+ o_orderkey: int32,
+ o_custkey: int32,
+ o_orderstatus: string,
+ o_totalprice: double,
+ o_orderdate: string,
+ o_orderpriority: string,
+ o_clerk: string,
+ o_shippriority: int32,
+ o_comment: string
+};
+
+create dataset Orders(OrderType) primary key o_orderkey;
+
+select *
+from Orders
+where o_totalprice > 100
+limit 10;
diff --git
a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/push-select-to-lookup.sqlpp
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/push-select-to-lookup.sqlpp
new file mode 100644
index 0000000..c34b534
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/push-select-to-lookup.sqlpp
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+drop dataverse tpch if exists;
+create dataverse tpch;
+use tpch;
+
+create type OrderType as closed {
+ o_orderkey: int32,
+ o_custkey: int32,
+ o_orderstatus: string,
+ o_totalprice: double,
+ o_orderdate: string,
+ o_orderpriority: string,
+ o_clerk: string,
+ o_shippriority: int32,
+ o_comment: string
+};
+
+create dataset Orders(OrderType) primary key o_orderkey;
+
+create index statusIndex on Orders(o_orderstatus) type btree;
+
+select *
+from Orders
+where o_totalprice > 100 AND o_orderstatus = "finished";
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/push-select-to-scan.sqlpp
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/push-select-to-scan.sqlpp
new file mode 100644
index 0000000..8751035
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/push-select-to-scan.sqlpp
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+drop dataverse tpch if exists;
+create dataverse tpch;
+use tpch;
+
+create type OrderType as closed {
+ o_orderkey: int32,
+ o_custkey: int32,
+ o_orderstatus: string,
+ o_totalprice: double,
+ o_orderdate: string,
+ o_orderpriority: string,
+ o_clerk: string,
+ o_shippriority: int32,
+ o_comment: string
+};
+
+create dataset Orders(OrderType) primary key o_orderkey;
+
+select *
+from Orders
+where o_totalprice > 100;
diff --git
a/asterixdb/asterix-app/src/test/resources/optimizerts/results/push-limit-to-lookup.plan
b/asterixdb/asterix-app/src/test/resources/optimizerts/results/push-limit-to-lookup.plan
new file mode 100644
index 0000000..8301c01
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/optimizerts/results/push-limit-to-lookup.plan
@@ -0,0 +1,20 @@
+-- DISTRIBUTE_RESULT |UNPARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED|
+ -- STREAM_LIMIT |UNPARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$17(ASC) ] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_LIMIT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$23(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git
a/asterixdb/asterix-app/src/test/resources/optimizerts/results/push-limit-to-scan.plan
b/asterixdb/asterix-app/src/test/resources/optimizerts/results/push-limit-to-scan.plan
new file mode 100644
index 0000000..e6e5511
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/optimizerts/results/push-limit-to-scan.plan
@@ -0,0 +1,13 @@
+-- DISTRIBUTE_RESULT |UNPARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED|
+ -- STREAM_LIMIT |UNPARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$14(ASC) ] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_LIMIT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git
a/asterixdb/asterix-app/src/test/resources/optimizerts/results/push-select-to-lookup.plan
b/asterixdb/asterix-app/src/test/resources/optimizerts/results/push-select-to-lookup.plan
new file mode 100644
index 0000000..7d83a8d
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/optimizerts/results/push-select-to-lookup.plan
@@ -0,0 +1,17 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$23(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git
a/asterixdb/asterix-app/src/test/resources/optimizerts/results/push-select-to-scan.plan
b/asterixdb/asterix-app/src/test/resources/optimizerts/results/push-select-to-scan.plan
new file mode 100644
index 0000000..ede583b
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/optimizerts/results/push-select-to-scan.plan
@@ -0,0 +1,10 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-scan-select/limit-scan-select.1.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-scan-select/limit-scan-select.1.ddl.sqlpp
new file mode 100644
index 0000000..74d7a15
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-scan-select/limit-scan-select.1.ddl.sqlpp
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/* scan and print a delimited text file */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.DBLPType as
+{
+ id : bigint,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+};
+
+create dataset DBLP1(DBLPType) primary key id;
+
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-scan-select/limit-scan-select.2.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-scan-select/limit-scan-select.2.update.sqlpp
new file mode 100644
index 0000000..2e0f056
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-scan-select/limit-scan-select.2.update.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/* scan and print a delimited text file */
+
+use test;
+
+
+load dataset DBLP1 using localfs
((`path`=`asterix_nc1://data/dblp-small/dblp-small-id.txt`),(`format`=`delimited-text`),(`delimiter`=`:`));
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-scan-select/limit-scan-select.3.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-scan-select/limit-scan-select.3.query.sqlpp
new file mode 100644
index 0000000..657bfb4
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-scan-select/limit-scan-select.3.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/* scan and print a delimited text file */
+
+use test;
+
+explain
+ select element paper
+ from DBLP1 as paper
+ where contains(dblpid, 'kimL89')
+ limit 5 offset 5;
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-scan-select/limit-scan-select.4.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-scan-select/limit-scan-select.4.query.sqlpp
new file mode 100644
index 0000000..6459b5c
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-scan-select/limit-scan-select.4.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/* scan and print a delimited text file */
+
+use test;
+
+select element paper
+from DBLP1 as paper
+where contains(dblpid, 'kimL89')
+limit 5 offset 5;
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-scan/limit-scan.1.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-scan/limit-scan.1.ddl.sqlpp
new file mode 100644
index 0000000..74d7a15
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-scan/limit-scan.1.ddl.sqlpp
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/* scan and print a delimited text file */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.DBLPType as
+{
+ id : bigint,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+};
+
+create dataset DBLP1(DBLPType) primary key id;
+
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-scan/limit-scan.2.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-scan/limit-scan.2.update.sqlpp
new file mode 100644
index 0000000..2e0f056
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-scan/limit-scan.2.update.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/* scan and print a delimited text file */
+
+use test;
+
+
+load dataset DBLP1 using localfs
((`path`=`asterix_nc1://data/dblp-small/dblp-small-id.txt`),(`format`=`delimited-text`),(`delimiter`=`:`));
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-scan/limit-scan.3.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-scan/limit-scan.3.query.sqlpp
new file mode 100644
index 0000000..d2cb38a
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-scan/limit-scan.3.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/* scan and print a delimited text file */
+
+use test;
+
+explain
+ select element paper
+ from DBLP1 as paper
+ limit 5 offset 5;
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-scan/limit-scan.4.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-scan/limit-scan.4.query.sqlpp
new file mode 100644
index 0000000..5b7c5be
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-scan/limit-scan.4.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/* scan and print a delimited text file */
+
+use test;
+
+select element paper
+from DBLP1 as paper
+limit 5 offset 5;
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-secondary-index/limit-secondary-index.1.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-secondary-index/limit-secondary-index.1.ddl.sqlpp
new file mode 100644
index 0000000..8a5a54c
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-secondary-index/limit-secondary-index.1.ddl.sqlpp
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : using-constant-merge-policy.aql
+ * Description : This test is intended to test the compact statement which
merge the disk components of a dataset and
+ * all of its indexes using the constant merge policy.
+ * Expected Result : Success
+ * Date : Sep 19 2013
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.LineItemType as
+ closed {
+ l_orderkey : bigint,
+ l_partkey : bigint,
+ l_suppkey : bigint,
+ l_linenumber : bigint,
+ l_quantity : bigint,
+ l_extendedprice : double,
+ l_discount : double,
+ l_tax : double,
+ l_returnflag : string,
+ l_linestatus : string,
+ l_shipdate : string,
+ l_commitdate : string,
+ l_receiptdate : string,
+ l_shipinstruct : string,
+ l_shipmode : string,
+ l_comment : string
+};
+
+create dataset LineItem(LineItemType) primary key l_orderkey,l_linenumber;
+
+create index idx_LineItem_suppkey on LineItem (l_suppkey) type btree;
+
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-secondary-index/limit-secondary-index.2.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-secondary-index/limit-secondary-index.2.update.sqlpp
new file mode 100644
index 0000000..546a831
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-secondary-index/limit-secondary-index.2.update.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+load dataset LineItem using localfs
((`path`=`asterix_nc1://data/tpch0.001/lineitem.tbl`),(`format`=`delimited-text`),(`delimiter`=`|`))
pre-sorted;
+
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-secondary-index/limit-secondary-index.3.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-secondary-index/limit-secondary-index.3.query.sqlpp
new file mode 100644
index 0000000..43f7d94
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-secondary-index/limit-secondary-index.3.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+explain select element c
+from LineItem as c
+where (c.l_suppkey < 150)
+limit 5 offset 5;
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-secondary-index/limit-secondary-index.4.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-secondary-index/limit-secondary-index.4.query.sqlpp
new file mode 100644
index 0000000..026aed7
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/limit/limit-secondary-index/limit-secondary-index.4.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+select element c
+from LineItem as c
+where (c.l_suppkey < 150)
+limit 5 offset 5;
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/limit/limit-scan-select/limit-scan-select.3.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/limit/limit-scan-select/limit-scan-select.3.adm
new file mode 100644
index 0000000..8f202b7
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/limit/limit-scan-select/limit-scan-select.3.adm
@@ -0,0 +1,22 @@
+distribute result [$$paper]
+-- DISTRIBUTE_RESULT |UNPARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED|
+ limit 5, 5
+ -- STREAM_LIMIT |UNPARTITIONED|
+ project ([$$paper])
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange
+ -- SORT_MERGE_EXCHANGE [$$12(ASC) ] |PARTITIONED|
+ limit 10
+ -- STREAM_LIMIT |PARTITIONED|
+ select (contains($$paper.getField(1), "kimL89"))
+ -- STREAM_SELECT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$12, $$paper] <- test.DBLP1 select condition
(contains($$paper.getField(1), "kimL89")) output limit 10
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/limit/limit-scan-select/limit-scan-select.4.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/limit/limit-scan-select/limit-scan-select.4.adm
new file mode 100644
index 0000000..afc422a
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/limit/limit-scan-select/limit-scan-select.4.adm
@@ -0,0 +1,5 @@
+{ "id": 41, "dblpid": "books/aw/kimL89/EllisG89", "title": "Active Objects
Ealities and Possibilities.", "authors": "Clarence A. Ellis Simon J. Gibbs",
"misc": "2002-01-03 561-572 1989 Object-Oriented Concepts, Databases, and
Applications db/books/collections/kim89.html#EllisG89" }
+{ "id": 42, "dblpid": "books/aw/kimL89/FishmanABCCDHHKLLMNRSW89", "title":
"Overview of the Iris DBMS.", "authors": "Daniel H. Fishman Jurgen Annevelink
David Beech E. C. Chow Tim Connors J. W. Davis Waqar Hasan C. G. Hoch William
Kent S. Leichner Peter Lyngbæk Brom Mahbod Marie-Anne Neimat Tore Risch
Ming-Chien Shan W. Kevin Wilkinson", "misc": "2002-01-03 219-250
Object-Oriented Concepts, Databases, and Applications ACM Press and
Addison-Wesley 1989 db/books/collections/kim89.html#FishmanABCCDHHKLLMNRSW89" }
+{ "id": 43, "dblpid": "books/aw/kimL89/KimBCGW89", "title": "Features of the
ORION Object-Oriented Database System.", "authors": "Won Kim Nat Ballou
Hong-Tai Chou Jorge F. Garza Darrell Woelk", "misc": "2002-01-03 251-282
Object-Oriented Concepts, Databases, and Applications ACM Press and
Addison-Wesley 1989 db/books/collections/kim89.html#KimBCGW89" }
+{ "id": 44, "dblpid": "books/aw/kimL89/KimKD89", "title": "Indexing Techniques
for Object-Oriented Databases.", "authors": "Won Kim Kyung-Chang Kim Alfred G.
Dale", "misc": "2002-01-03 371-394 Object-Oriented Concepts, Databases, and
Applications ACM Press and Addison-Wesley 1989
db/books/collections/kim89.html#KimKD89" }
+{ "id": 45, "dblpid": "books/aw/kimL89/King89", "title": "My Cat Is
Object-Oriented.", "authors": "Roger King", "misc": "2002-01-03 23-30 1989
Object-Oriented Concepts, Databases, and Applications
db/books/collections/kim89.html#King89" }
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/limit/limit-scan/limit-scan.3.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/limit/limit-scan/limit-scan.3.adm
new file mode 100644
index 0000000..24d2d15
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/limit/limit-scan/limit-scan.3.adm
@@ -0,0 +1,20 @@
+distribute result [$$paper]
+-- DISTRIBUTE_RESULT |UNPARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED|
+ limit 5, 5
+ -- STREAM_LIMIT |UNPARTITIONED|
+ project ([$$paper])
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange
+ -- SORT_MERGE_EXCHANGE [$$10(ASC) ] |PARTITIONED|
+ limit 10
+ -- STREAM_LIMIT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$10, $$paper] <- test.DBLP1 output limit 10
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/limit/limit-scan/limit-scan.4.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/limit/limit-scan/limit-scan.4.adm
new file mode 100644
index 0000000..e9f7751
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/limit/limit-scan/limit-scan.4.adm
@@ -0,0 +1,5 @@
+{ "id": 6, "dblpid": "books/acm/kim95/DittrichD95", "title": "Where
Object-Oriented DBMSs Should Do Better A Critique Based on Early
Experiences.", "authors": "Angelika Kotz Dittrich Klaus R. Dittrich", "misc":
"2002-01-03 238-254 1995 Modern Database Systems
db/books/collections/kim95.html#DittrichD95" }
+{ "id": 7, "dblpid": "books/acm/kim95/Garcia-MolinaH95", "title": "Distributed
Databases.", "authors": "Hector Garcia-Molina Meichun Hsu", "misc": "2002-01-03
477-493 1995 Modern Database Systems
db/books/collections/kim95.html#Garcia-MolinaH95" }
+{ "id": 8, "dblpid": "books/acm/kim95/Goodman95", "title": "An Object-Oriented
DBMS War Story Developing a Genome Mapping Database in C++.", "authors":
"Nathan Goodman", "misc": "2002-01-03 216-237 1995 Modern Database Systems
db/books/collections/kim95.html#Goodman95" }
+{ "id": 9, "dblpid": "books/acm/kim95/Kaiser95", "title": "Cooperative
Transactions for Multiuser Environments.", "authors": "Gail E. Kaiser", "misc":
"2002-01-03 409-433 1995 Modern Database Systems
db/books/collections/kim95.html#Kaiser95" }
+{ "id": 10, "dblpid": "books/acm/kim95/KelleyGKRG95", "title": "Schema
Architecture of the UniSQL/M Multidatabase System", "authors": "William Kelley
Sunit K. Gala Won Kim Tom C. Reyes Bruce Graham", "misc": "2004-03-08 Modern
Database Systems books/acm/Kim95 621-648 1995
db/books/collections/kim95.html#KelleyGKRG95" }
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/limit/limit-secondary-index/limit-secondary-index.3.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/limit/limit-secondary-index/limit-secondary-index.3.adm
new file mode 100644
index 0000000..6234992
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/limit/limit-secondary-index/limit-secondary-index.3.adm
@@ -0,0 +1,36 @@
+distribute result [$$c]
+-- DISTRIBUTE_RESULT |UNPARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED|
+ limit 5, 5
+ -- STREAM_LIMIT |UNPARTITIONED|
+ project ([$$c])
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange
+ -- SORT_MERGE_EXCHANGE [$$12(ASC), $$13(ASC) ] |PARTITIONED|
+ limit 10
+ -- STREAM_LIMIT |PARTITIONED|
+ select (lt($$c.getField(2), 150))
+ -- STREAM_SELECT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ unnest-map [$$12, $$13, $$c] <- index-search("LineItem", 0,
"test", "LineItem", FALSE, FALSE, 2, $$17, $$18, 2, $$17, $$18, TRUE, TRUE,
TRUE) select condition (lt($$c.getField(2), 150)) output limit 10
+ -- BTREE_SEARCH |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ order (ASC, $$17) (ASC, $$18)
+ -- STABLE_SORT [$$17(ASC), $$18(ASC)] |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$17, $$18])
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ unnest-map [$$16, $$17, $$18] <-
index-search("idx_LineItem_suppkey", 0, "test", "LineItem", FALSE, FALSE, 0, 1,
$$15, TRUE, FALSE, FALSE)
+ -- BTREE_SEARCH |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ assign [$$15] <- [150]
+ -- ASSIGN |PARTITIONED|
+ empty-tuple-source
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/limit/limit-secondary-index/limit-secondary-index.4.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/limit/limit-secondary-index/limit-secondary-index.4.adm
new file mode 100644
index 0000000..eeedd56
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/limit/limit-secondary-index/limit-secondary-index.4.adm
@@ -0,0 +1,5 @@
+{ "l_orderkey": 1, "l_partkey": 16, "l_suppkey": 3, "l_linenumber": 6,
"l_quantity": 32, "l_extendedprice": 29312.32, "l_discount": 0.07, "l_tax":
0.02, "l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1996-01-30",
"l_commitdate": "1996-02-07", "l_receiptdate": "1996-02-03", "l_shipinstruct":
"DELIVER IN PERSON", "l_shipmode": "MAIL", "l_comment": "arefully slyly ex" }
+{ "l_orderkey": 2, "l_partkey": 107, "l_suppkey": 2, "l_linenumber": 1,
"l_quantity": 38, "l_extendedprice": 38269.8, "l_discount": 0.0, "l_tax": 0.05,
"l_returnflag": "N", "l_linestatus": "O", "l_shipdate": "1997-01-28",
"l_commitdate": "1997-01-14", "l_receiptdate": "1997-02-02", "l_shipinstruct":
"TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": "ven requests. deposits
breach a" }
+{ "l_orderkey": 3, "l_partkey": 5, "l_suppkey": 2, "l_linenumber": 1,
"l_quantity": 45, "l_extendedprice": 40725.0, "l_discount": 0.06, "l_tax": 0.0,
"l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-02-02",
"l_commitdate": "1994-01-04", "l_receiptdate": "1994-02-23", "l_shipinstruct":
"NONE", "l_shipmode": "AIR", "l_comment": "ongside of the furiously brave acco"
}
+{ "l_orderkey": 3, "l_partkey": 20, "l_suppkey": 10, "l_linenumber": 2,
"l_quantity": 49, "l_extendedprice": 45080.98, "l_discount": 0.1, "l_tax": 0.0,
"l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1993-11-09",
"l_commitdate": "1993-12-20", "l_receiptdate": "1993-11-24", "l_shipinstruct":
"TAKE BACK RETURN", "l_shipmode": "RAIL", "l_comment": " unusual accounts. eve"
}
+{ "l_orderkey": 3, "l_partkey": 129, "l_suppkey": 8, "l_linenumber": 3,
"l_quantity": 27, "l_extendedprice": 27786.24, "l_discount": 0.06, "l_tax":
0.07, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-01-16",
"l_commitdate": "1993-11-22", "l_receiptdate": "1994-01-23", "l_shipinstruct":
"DELIVER IN PERSON", "l_shipmode": "SHIP", "l_comment": "nal foxes wake. " }
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 95c059d..cb07849 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -9760,5 +9760,22 @@
</compilation-unit>
</test-case>
</test-group>
+ <test-group name="limit">
+ <test-case FilePath="limit">
+ <compilation-unit name="limit-scan">
+ <output-dir compare="Text">limit-scan</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="limit">
+ <compilation-unit name="limit-scan-select">
+ <output-dir compare="Text">limit-scan-select</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="limit">
+ <compilation-unit name="limit-secondary-index">
+ <output-dir compare="Text">limit-secondary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
&GeoQueries;
</test-suite>
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DataSource.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DataSource.java
index 66283d6..88449f9 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DataSource.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DataSource.java
@@ -40,6 +40,7 @@
import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
import org.apache.hyracks.api.job.JobSpecification;
+import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
public abstract class DataSource implements IDataSource<DataSourceId> {
@@ -159,7 +160,8 @@
public abstract Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>
buildDatasourceScanRuntime(
MetadataProvider metadataProvider, IDataSource<DataSourceId>
dataSource,
List<LogicalVariable> scanVariables, List<LogicalVariable>
projectVariables, boolean projectPushed,
- List<LogicalVariable> minFilterVars, List<LogicalVariable>
maxFilterVars, IOperatorSchema opSchema,
+ List<LogicalVariable> minFilterVars, List<LogicalVariable>
maxFilterVars,
+ ITupleFilterFactory tupleFilterFactory, long outputLimit,
IOperatorSchema opSchema,
IVariableTypeEnvironment typeEnv, JobGenContext context,
JobSpecification jobSpec, Object implConfig)
throws AlgebricksException;
}
\ No newline at end of file
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
index ddab5bc..953a63a 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
@@ -42,6 +42,7 @@
import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
import org.apache.hyracks.api.job.JobSpecification;
+import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
public class DatasetDataSource extends DataSource {
@@ -92,7 +93,8 @@
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>
buildDatasourceScanRuntime(
MetadataProvider metadataProvider, IDataSource<DataSourceId>
dataSource,
List<LogicalVariable> scanVariables, List<LogicalVariable>
projectVariables, boolean projectPushed,
- List<LogicalVariable> minFilterVars, List<LogicalVariable>
maxFilterVars, IOperatorSchema opSchema,
+ List<LogicalVariable> minFilterVars, List<LogicalVariable>
maxFilterVars,
+ ITupleFilterFactory selectCondition, long outputLimit,
IOperatorSchema opSchema,
IVariableTypeEnvironment typeEnv, JobGenContext context,
JobSpecification jobSpec, Object implConfig)
throws AlgebricksException {
switch (dataset.getDatasetType()) {
@@ -117,7 +119,7 @@
int[] maxFilterFieldIndexes =
createFilterIndexes(maxFilterVars, opSchema);
return metadataProvider.buildBtreeRuntime(jobSpec, opSchema,
typeEnv, context, true, false,
((DatasetDataSource) dataSource).getDataset(),
primaryIndex.getIndexName(), null, null, true,
- true, false, minFilterFieldIndexes,
maxFilterFieldIndexes, false);
+ true, false, minFilterFieldIndexes,
maxFilterFieldIndexes, selectCondition, outputLimit, false);
default:
throw new AlgebricksException("Unknown datasource type");
}
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
index 5c3ed56..5827d38 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
@@ -22,7 +22,6 @@
import java.util.List;
import org.apache.asterix.active.EntityId;
-import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.external.feed.management.FeedConnectionId;
import org.apache.asterix.external.operators.FeedCollectOperatorDescriptor;
import org.apache.asterix.external.util.FeedUtils.FeedRuntimeType;
@@ -48,6 +47,7 @@
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.job.JobSpecification;
+import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
public class FeedDataSource extends DataSource implements IMutationDataSource {
@@ -158,7 +158,8 @@
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>
buildDatasourceScanRuntime(
MetadataProvider metadataProvider, IDataSource<DataSourceId>
dataSource,
List<LogicalVariable> scanVariables, List<LogicalVariable>
projectVariables, boolean projectPushed,
- List<LogicalVariable> minFilterVars, List<LogicalVariable>
maxFilterVars, IOperatorSchema opSchema,
+ List<LogicalVariable> minFilterVars, List<LogicalVariable>
maxFilterVars,
+ ITupleFilterFactory tupleFilterFactory, long outputLimit,
IOperatorSchema opSchema,
IVariableTypeEnvironment typeEnv, JobGenContext context,
JobSpecification jobSpec, Object implConfig)
throws AlgebricksException {
try {
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
index d2b9871..24ad354 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
@@ -44,6 +44,7 @@
import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
import org.apache.hyracks.api.job.JobSpecification;
+import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
public abstract class FunctionDataSource extends DataSource {
@@ -68,7 +69,8 @@
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>
buildDatasourceScanRuntime(
MetadataProvider metadataProvider, IDataSource<DataSourceId>
dataSource,
List<LogicalVariable> scanVariables, List<LogicalVariable>
projectVariables, boolean projectPushed,
- List<LogicalVariable> minFilterVars, List<LogicalVariable>
maxFilterVars, IOperatorSchema opSchema,
+ List<LogicalVariable> minFilterVars, List<LogicalVariable>
maxFilterVars,
+ ITupleFilterFactory tupeFilterFactory, long outputLimit,
IOperatorSchema opSchema,
IVariableTypeEnvironment typeEnv, JobGenContext context,
JobSpecification jobSpec, Object implConfig)
throws AlgebricksException {
GenericAdapterFactory adapterFactory = new GenericAdapterFactory();
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java
index 91030bf..3fb2230 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java
@@ -44,6 +44,7 @@
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.JobSpecification;
+import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
public class LoadableDataSource extends DataSource {
@@ -124,7 +125,8 @@
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>
buildDatasourceScanRuntime(
MetadataProvider metadataProvider, IDataSource<DataSourceId>
dataSource,
List<LogicalVariable> scanVariables, List<LogicalVariable>
projectVariables, boolean projectPushed,
- List<LogicalVariable> minFilterVars, List<LogicalVariable>
maxFilterVars, IOperatorSchema opSchema,
+ List<LogicalVariable> minFilterVars, List<LogicalVariable>
maxFilterVars,
+ ITupleFilterFactory tupeFilterFactory, long outputLimit,
IOperatorSchema opSchema,
IVariableTypeEnvironment typeEnv, JobGenContext context,
JobSpecification jobSpec, Object implConfig)
throws AlgebricksException {
LoadableDataSource alds = (LoadableDataSource) dataSource;
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index a081fb4..19030d3 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -26,7 +26,6 @@
import java.util.List;
import java.util.Map;
-import org.apache.asterix.common.transactions.ITxnIdFactory;
import org.apache.asterix.common.cluster.IClusterStateManager;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.DatasetConfig.ExternalFilePendingOp;
@@ -38,6 +37,7 @@
import
org.apache.asterix.common.dataflow.LSMTreeInsertDeleteOperatorDescriptor;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.metadata.LockList;
+import org.apache.asterix.common.transactions.ITxnIdFactory;
import org.apache.asterix.common.transactions.TxnId;
import org.apache.asterix.common.utils.StoragePathUtil;
import org.apache.asterix.dataflow.data.nontagged.MissingWriterFactory;
@@ -132,6 +132,7 @@
import
org.apache.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
import
org.apache.hyracks.storage.am.common.api.IModificationOperationCallbackFactory;
import
org.apache.hyracks.storage.am.common.api.ISearchOperationCallbackFactory;
+import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
import
org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import
org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
@@ -380,10 +381,12 @@
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>
getScannerRuntime(
IDataSource<DataSourceId> dataSource, List<LogicalVariable>
scanVariables,
List<LogicalVariable> projectVariables, boolean projectPushed,
List<LogicalVariable> minFilterVars,
- List<LogicalVariable> maxFilterVars, IOperatorSchema opSchema,
IVariableTypeEnvironment typeEnv,
- JobGenContext context, JobSpecification jobSpec, Object
implConfig) throws AlgebricksException {
+ List<LogicalVariable> maxFilterVars, ITupleFilterFactory
tupleFilterFactory, long outputLimit,
+ IOperatorSchema opSchema, IVariableTypeEnvironment typeEnv,
JobGenContext context, JobSpecification jobSpec,
+ Object implConfig) throws AlgebricksException {
return ((DataSource) dataSource).buildDatasourceScanRuntime(this,
dataSource, scanVariables, projectVariables,
- projectPushed, minFilterVars, maxFilterVars, opSchema,
typeEnv, context, jobSpec, implConfig);
+ projectPushed, minFilterVars, maxFilterVars,
tupleFilterFactory, outputLimit, opSchema, typeEnv,
+ context, jobSpec, implConfig);
}
protected Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>
buildLoadableDatasetScan(
@@ -433,7 +436,8 @@
IOperatorSchema opSchema, IVariableTypeEnvironment typeEnv,
JobGenContext context, boolean retainInput,
boolean retainMissing, Dataset dataset, String indexName, int[]
lowKeyFields, int[] highKeyFields,
boolean lowKeyInclusive, boolean highKeyInclusive, boolean
propagateFilter, int[] minFilterFieldIndexes,
- int[] maxFilterFieldIndexes, boolean isIndexOnlyPlan) throws
AlgebricksException {
+ int[] maxFilterFieldIndexes, ITupleFilterFactory
tupleFilterFactory, long outputLimit,
+ boolean isIndexOnlyPlan) throws AlgebricksException {
boolean isSecondary = true;
Index primaryIndex = MetadataManager.INSTANCE.getIndex(mdTxnCtx,
dataset.getDataverseName(),
dataset.getDatasetName(), dataset.getDatasetName());
@@ -477,8 +481,8 @@
btreeSearchOp = new BTreeSearchOperatorDescriptor(jobSpec,
outputRecDesc, lowKeyFields, highKeyFields,
lowKeyInclusive, highKeyInclusive, indexHelperFactory,
retainInput, retainMissing,
context.getMissingWriterFactory(), searchCallbackFactory,
minFilterFieldIndexes,
- maxFilterFieldIndexes, propagateFilter,
proceedIndexOnlyPlan, failValueForIndexOnlyPlan,
- successValueForIndexOnlyPlan);
+ maxFilterFieldIndexes, propagateFilter,
tupleFilterFactory, outputLimit, proceedIndexOnlyPlan,
+ failValueForIndexOnlyPlan, successValueForIndexOnlyPlan);
} else {
btreeSearchOp = new ExternalBTreeSearchOperatorDescriptor(jobSpec,
outputRecDesc, lowKeyFields,
highKeyFields, lowKeyInclusive, highKeyInclusive,
indexHelperFactory, retainInput, retainMissing,
@@ -1043,7 +1047,7 @@
prevAdditionalFilteringKeys = new ArrayList<>();
prevAdditionalFilteringKeys.add(prevAdditionalFilteringKey);
}
- AsterixTupleFilterFactory filterFactory =
createTupleFilterFactory(inputSchemas, typeEnv, filterExpr, context);
+ AsterixTupleFilterFactory filterFactory =
getTupleFilterFactory(inputSchemas, typeEnv, filterExpr, context);
switch (secondaryIndex.getIndexType()) {
case BTREE:
return getBTreeRuntime(dataverseName, datasetName, indexName,
propagatedSchema, primaryKeys,
@@ -1541,7 +1545,7 @@
}
}
- private AsterixTupleFilterFactory
createTupleFilterFactory(IOperatorSchema[] inputSchemas,
+ public AsterixTupleFilterFactory getTupleFilterFactory(IOperatorSchema[]
inputSchemas,
IVariableTypeEnvironment typeEnv, ILogicalExpression filterExpr,
JobGenContext context)
throws AlgebricksException {
// No filtering condition.
diff --git a/hyracks-fullstack/algebricks/algebricks-core/pom.xml
b/hyracks-fullstack/algebricks/algebricks-core/pom.xml
index 669988c..2a97b6f 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/pom.xml
+++ b/hyracks-fullstack/algebricks/algebricks-core/pom.xml
@@ -78,6 +78,11 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-storage-am-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
diff --git
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
index 6860147..a9744b5 100644
---
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
+++
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
@@ -36,6 +36,7 @@
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.job.JobSpecification;
+import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
public interface IMetadataProvider<S, I> {
public IDataSource<S> findDataSource(S id) throws AlgebricksException;
@@ -47,7 +48,8 @@
*/
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>
getScannerRuntime(IDataSource<S> dataSource,
List<LogicalVariable> scanVariables, List<LogicalVariable>
projectVariables, boolean projectPushed,
- List<LogicalVariable> minFilterVars, List<LogicalVariable>
maxFilterVars, IOperatorSchema opSchema,
+ List<LogicalVariable> minFilterVars, List<LogicalVariable>
maxFilterVars,
+ ITupleFilterFactory tupleFilterFactory, long outputLimit,
IOperatorSchema opSchema,
IVariableTypeEnvironment typeEnv, JobGenContext context,
JobSpecification jobSpec, Object implConfig)
throws AlgebricksException;
@@ -206,5 +208,8 @@
List<LogicalVariable> prevSecondaryKeys, LogicalVariable
prevAdditionalFilteringKeys,
RecordDescriptor inputDesc, JobGenContext context,
JobSpecification spec) throws AlgebricksException;
+ public ITupleFilterFactory getTupleFilterFactory(IOperatorSchema[]
inputSchemas, IVariableTypeEnvironment typeEnv,
+ ILogicalExpression filterExpr, JobGenContext context) throws
AlgebricksException;
+
public Map<String, String> getConfig();
}
diff --git
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java
index 2511fa9..b9700ab 100644
---
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java
+++
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java
@@ -35,13 +35,16 @@
import
org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
public class DataSourceScanOperator extends AbstractDataSourceOperator {
- private List<LogicalVariable> projectVars;
+ private final List<LogicalVariable> projectVars;
private boolean projectPushed = false;
private List<Mutable<ILogicalExpression>> additionalFilteringExpressions;
private List<LogicalVariable> minFilterVars;
private List<LogicalVariable> maxFilterVars;
+
+ private Mutable<ILogicalExpression> selectCondition;
+ private long outputLimit = -1;
public DataSourceScanOperator(List<LogicalVariable> variables,
IDataSource<?> dataSource) {
super(variables, dataSource);
@@ -133,4 +136,20 @@
public List<Mutable<ILogicalExpression>>
getAdditionalFilteringExpressions() {
return additionalFilteringExpressions;
}
+
+ public Mutable<ILogicalExpression> getSelectCondition() {
+ return selectCondition;
+ }
+
+ public void setSelectCondition(Mutable<ILogicalExpression>
selectCondition) {
+ this.selectCondition = selectCondition;
+ }
+
+ public long getOutputLimit() {
+ return outputLimit;
+ }
+
+ public void setOutputLimit(long outputLimit) {
+ this.outputLimit = outputLimit;
+ }
}
diff --git
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java
index e7fb6c0..0a5e76a 100644
---
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java
+++
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java
@@ -32,6 +32,9 @@
public class UnnestMapOperator extends AbstractUnnestMapOperator {
+ private Mutable<ILogicalExpression> selectCondition;
+ private long outputLimit = -1;
+
public UnnestMapOperator(List<LogicalVariable> variables,
Mutable<ILogicalExpression> expression,
List<Object> variableTypes, boolean propagateInput) {
super(variables, expression, variableTypes, propagateInput);
@@ -64,4 +67,20 @@
return env;
}
+ public Mutable<ILogicalExpression> getSelectCondition() {
+ return selectCondition;
+ }
+
+ public void setSelectCondition(Mutable<ILogicalExpression>
selectCondition) {
+ this.selectCondition = selectCondition;
+ }
+
+ public long getOutputLimit() {
+ return outputLimit;
+ }
+
+ public void setOutputLimit(long outputLimit) {
+ this.outputLimit = outputLimit;
+ }
+
}
diff --git
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java
index b8cb4ff..b15edd2 100644
---
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java
+++
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java
@@ -34,11 +34,11 @@
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import
org.apache.hyracks.algebricks.core.algebra.operators.logical.DelegateOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
-import
org.apache.hyracks.algebricks.core.algebra.operators.logical.DelegateOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteUpsertOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
@@ -76,7 +76,7 @@
public class UsedVariableVisitor implements ILogicalOperatorVisitor<Void,
Void> {
- private Collection<LogicalVariable> usedVariables;
+ private final Collection<LogicalVariable> usedVariables;
public UsedVariableVisitor(Collection<LogicalVariable> usedVariables) {
this.usedVariables = usedVariables;
@@ -305,6 +305,9 @@
@Override
public Void visitUnnestMapOperator(UnnestMapOperator op, Void arg) {
getUsedVarsFromExprAndFilterExpr(op);
+ if (op.getSelectCondition() != null) {
+ op.getSelectCondition().getValue().getUsedVariables(usedVariables);
+ }
return null;
}
diff --git
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java
index 1421cef..5d90bd1 100644
---
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java
+++
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java
@@ -43,6 +43,7 @@
import
org.apache.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
+import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
@SuppressWarnings("rawtypes")
public class DataSourceScanPOperator extends AbstractScanPOperator {
@@ -109,9 +110,15 @@
List<LogicalVariable> vars = scan.getVariables();
List<LogicalVariable> projectVars = scan.getProjectVariables();
- Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> p =
- mp.getScannerRuntime(dataSource, vars, projectVars,
scan.isProjectPushed(), scan.getMinFilterVars(),
- scan.getMaxFilterVars(), opSchema, typeEnv, context,
builder.getJobSpec(), implConfig);
+ ITupleFilterFactory tupleFilterFactory = null;
+ if (scan.getSelectCondition() != null) {
+ tupleFilterFactory =
context.getMetadataProvider().getTupleFilterFactory(new IOperatorSchema[] {
opSchema },
+ typeEnv, scan.getSelectCondition().getValue(), context);
+ }
+
+ Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> p =
mp.getScannerRuntime(dataSource, vars, projectVars,
+ scan.isProjectPushed(), scan.getMinFilterVars(),
scan.getMaxFilterVars(), tupleFilterFactory,
+ scan.getOutputLimit(), opSchema, typeEnv, context,
builder.getJobSpec(), implConfig);
builder.contributeHyracksOperator(scan, p.first);
if (p.second != null) {
builder.contributeAlgebricksPartitionConstraint(p.first, p.second);
diff --git
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java
index 70f19c1..aeeb21a 100644
---
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java
+++
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java
@@ -299,21 +299,25 @@
@Override
public Void visitUnnestMapOperator(UnnestMapOperator op, Integer indent)
throws AlgebricksException {
- return printAbstractUnnestMapOperator(op, indent, "unnest-map");
+ AlgebricksAppendable plan = printAbstractUnnestMapOperator(op, indent,
"unnest-map");
+ appendSelectConditionInformation(plan, op.getSelectCondition(),
indent);
+ appendLimitInformation(plan, op.getOutputLimit());
+ return null;
}
@Override
public Void visitLeftOuterUnnestMapOperator(LeftOuterUnnestMapOperator op,
Integer indent)
throws AlgebricksException {
- return printAbstractUnnestMapOperator(op, indent,
"left-outer-unnest-map");
+ printAbstractUnnestMapOperator(op, indent, "left-outer-unnest-map");
+ return null;
}
- private Void printAbstractUnnestMapOperator(AbstractUnnestMapOperator op,
Integer indent, String opSignature)
- throws AlgebricksException {
+ private AlgebricksAppendable
printAbstractUnnestMapOperator(AbstractUnnestMapOperator op, Integer indent,
+ String opSignature) throws AlgebricksException {
AlgebricksAppendable plan = addIndent(indent).append(opSignature + " "
+ op.getVariables() + " <- "
+ op.getExpressionRef().getValue().accept(exprVisitor,
indent));
appendFilterInformation(plan, op.getMinFilterVars(),
op.getMaxFilterVars());
- return null;
+ return plan;
}
@Override
@@ -321,6 +325,25 @@
AlgebricksAppendable plan = addIndent(indent).append(
"data-scan " + op.getProjectVariables() + "<-" +
op.getVariables() + " <- " + op.getDataSource());
appendFilterInformation(plan, op.getMinFilterVars(),
op.getMaxFilterVars());
+ appendSelectConditionInformation(plan, op.getSelectCondition(),
indent);
+ appendLimitInformation(plan, op.getOutputLimit());
+ return null;
+ }
+
+ private Void appendSelectConditionInformation(AlgebricksAppendable plan,
+ Mutable<ILogicalExpression> selectCondition, Integer indent)
throws AlgebricksException {
+ if (selectCondition != null) {
+ plan.append(" select condition
(").append(selectCondition.getValue().accept(exprVisitor, indent))
+ .append(")");
+ }
+
+ return null;
+ }
+
+ private Void appendLimitInformation(AlgebricksAppendable plan, long
outputLimit) throws AlgebricksException {
+ if (outputLimit >= 0) {
+ plan.append(" output limit ").append(String.valueOf(outputLimit));
+ }
return null;
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorDescriptor.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorDescriptor.java
index 49eca6f..566d8e2 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorDescriptor.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorDescriptor.java
@@ -27,6 +27,7 @@
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import
org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import
org.apache.hyracks.storage.am.common.api.ISearchOperationCallbackFactory;
+import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
import
org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
public class BTreeSearchOperatorDescriptor extends
AbstractSingleActivityOperatorDescriptor {
@@ -48,6 +49,8 @@
protected boolean appendOpCallbackProceedResult;
protected byte[] searchCallbackProceedResultFalseValue;
protected byte[] searchCallbackProceedResultTrueValue;
+ protected final ITupleFilterFactory tupleFilterFactory;
+ protected final long outputLimit;
public BTreeSearchOperatorDescriptor(IOperatorDescriptorRegistry spec,
RecordDescriptor outRecDesc,
int[] lowKeyFields, int[] highKeyFields, boolean lowKeyInclusive,
boolean highKeyInclusive,
@@ -56,7 +59,7 @@
int[] minFilterFieldIndexes, int[] maxFilterFieldIndexes, boolean
appendIndexFilter) {
this(spec, outRecDesc, lowKeyFields, highKeyFields, lowKeyInclusive,
highKeyInclusive, indexHelperFactory,
retainInput, retainMissing, missingWriterFactory,
searchCallbackFactory, minFilterFieldIndexes,
- maxFilterFieldIndexes, appendIndexFilter, false, null, null);
+ maxFilterFieldIndexes, appendIndexFilter, null, -1, false,
null, null);
}
public BTreeSearchOperatorDescriptor(IOperatorDescriptorRegistry spec,
RecordDescriptor outRecDesc,
@@ -64,8 +67,8 @@
IIndexDataflowHelperFactory indexHelperFactory, boolean
retainInput, boolean retainMissing,
IMissingWriterFactory missingWriterFactory,
ISearchOperationCallbackFactory searchCallbackFactory,
int[] minFilterFieldIndexes, int[] maxFilterFieldIndexes, boolean
appendIndexFilter,
- boolean appendOpCallbackProceedResult, byte[]
searchCallbackProceedResultFalseValue,
- byte[] searchCallbackProceedResultTrueValue) {
+ ITupleFilterFactory tupleFilterFactory, long outputLimit, boolean
appendOpCallbackProceedResult,
+ byte[] searchCallbackProceedResultFalseValue, byte[]
searchCallbackProceedResultTrueValue) {
super(spec, 1, 1);
this.indexHelperFactory = indexHelperFactory;
this.retainInput = retainInput;
@@ -80,6 +83,8 @@
this.maxFilterFieldIndexes = maxFilterFieldIndexes;
this.appendIndexFilter = appendIndexFilter;
this.outRecDescs[0] = outRecDesc;
+ this.tupleFilterFactory = tupleFilterFactory;
+ this.outputLimit = outputLimit;
this.appendOpCallbackProceedResult = appendOpCallbackProceedResult;
this.searchCallbackProceedResultFalseValue =
searchCallbackProceedResultFalseValue;
this.searchCallbackProceedResultTrueValue =
searchCallbackProceedResultTrueValue;
@@ -92,7 +97,7 @@
recordDescProvider.getInputRecordDescriptor(getActivityId(),
0), lowKeyFields, highKeyFields,
lowKeyInclusive, highKeyInclusive, minFilterFieldIndexes,
maxFilterFieldIndexes, indexHelperFactory,
retainInput, retainMissing, missingWriterFactory,
searchCallbackFactory, appendIndexFilter,
- appendOpCallbackProceedResult,
searchCallbackProceedResultFalseValue,
+ tupleFilterFactory, outputLimit,
appendOpCallbackProceedResult, searchCallbackProceedResultFalseValue,
searchCallbackProceedResultTrueValue);
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
index 6bbf437..70b88fe 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
@@ -26,6 +26,7 @@
import org.apache.hyracks.storage.am.btree.util.BTreeUtils;
import
org.apache.hyracks.storage.am.common.api.ISearchOperationCallbackFactory;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
+import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
import
org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import
org.apache.hyracks.storage.am.common.dataflow.IndexSearchOperatorNodePushable;
import
org.apache.hyracks.storage.am.common.tuples.PermutingFrameTupleReference;
@@ -50,7 +51,7 @@
throws HyracksDataException {
this(ctx, partition, inputRecDesc, lowKeyFields, highKeyFields,
lowKeyInclusive, highKeyInclusive,
minFilterFieldIndexes, maxFilterFieldIndexes,
indexHelperFactory, retainInput, retainMissing,
- missingWriterFactory, searchCallbackFactory,
appendIndexFilter, false, null, null);
+ missingWriterFactory, searchCallbackFactory,
appendIndexFilter, null, -1, false, null, null);
}
public BTreeSearchOperatorNodePushable(IHyracksTaskContext ctx, int
partition, RecordDescriptor inputRecDesc,
@@ -58,11 +59,12 @@
int[] minFilterFieldIndexes, int[] maxFilterFieldIndexes,
IIndexDataflowHelperFactory indexHelperFactory,
boolean retainInput, boolean retainMissing, IMissingWriterFactory
missingWriterFactory,
ISearchOperationCallbackFactory searchCallbackFactory, boolean
appendIndexFilter,
- boolean appendOpCallbackProceedResult, byte[]
searchCallbackProceedResultFalseValue,
- byte[] searchCallbackProceedResultTrueValue) throws
HyracksDataException {
+ ITupleFilterFactory tupleFilterFactory, long outputLimit, boolean
appendOpCallbackProceedResult,
+ byte[] searchCallbackProceedResultFalseValue, byte[]
searchCallbackProceedResultTrueValue)
+ throws HyracksDataException {
super(ctx, inputRecDesc, partition, minFilterFieldIndexes,
maxFilterFieldIndexes, indexHelperFactory,
retainInput, retainMissing, missingWriterFactory,
searchCallbackFactory, appendIndexFilter,
- appendOpCallbackProceedResult,
searchCallbackProceedResultFalseValue,
+ tupleFilterFactory, outputLimit,
appendOpCallbackProceedResult, searchCallbackProceedResultFalseValue,
searchCallbackProceedResultTrueValue);
this.lowKeyInclusive = lowKeyInclusive;
this.highKeyInclusive = highKeyInclusive;
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
index 2c691f1..c88492b 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
@@ -22,6 +22,7 @@
import java.io.IOException;
import java.nio.ByteBuffer;
+import org.apache.hyracks.api.comm.IFrameTupleAccessor;
import org.apache.hyracks.api.comm.VSizeFrame;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.IMissingWriter;
@@ -37,11 +38,14 @@
import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender;
import org.apache.hyracks.dataflow.common.comm.util.FrameUtils;
import org.apache.hyracks.dataflow.common.data.accessors.FrameTupleReference;
+import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import
org.apache.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
import org.apache.hyracks.storage.am.common.api.IIndexDataflowHelper;
import org.apache.hyracks.storage.am.common.api.ILSMIndexCursor;
import
org.apache.hyracks.storage.am.common.api.ISearchOperationCallbackFactory;
+import org.apache.hyracks.storage.am.common.api.ITupleFilter;
+import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
import org.apache.hyracks.storage.am.common.impls.IndexAccessParameters;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
import
org.apache.hyracks.storage.am.common.tuples.PermutingFrameTupleReference;
@@ -95,22 +99,41 @@
protected byte[] searchCallbackProceedResultFalseValue;
protected byte[] searchCallbackProceedResultTrueValue;
+ protected final ITupleFilterFactory tupleFilterFactory;
+ protected ReferenceFrameTupleReference referenceFilterTuple;
+ protected ITupleFilter tupleFilter;
+ protected final long outputLimit;
+ protected long outputCount = 0;
+ protected boolean finished;
+
+ // no filter and limit pushdown
public IndexSearchOperatorNodePushable(IHyracksTaskContext ctx,
RecordDescriptor inputRecDesc, int partition,
int[] minFilterFieldIndexes, int[] maxFilterFieldIndexes,
IIndexDataflowHelperFactory indexHelperFactory,
boolean retainInput, boolean retainMissing, IMissingWriterFactory
missingWriterFactory,
ISearchOperationCallbackFactory searchCallbackFactory, boolean
appendIndexFilter)
throws HyracksDataException {
this(ctx, inputRecDesc, partition, minFilterFieldIndexes,
maxFilterFieldIndexes, indexHelperFactory,
- retainInput, retainMissing, missingWriterFactory,
searchCallbackFactory, appendIndexFilter, false, null,
- null);
+ retainInput, retainMissing, missingWriterFactory,
searchCallbackFactory, appendIndexFilter, null, -1,
+ false, null, null);
}
public IndexSearchOperatorNodePushable(IHyracksTaskContext ctx,
RecordDescriptor inputRecDesc, int partition,
int[] minFilterFieldIndexes, int[] maxFilterFieldIndexes,
IIndexDataflowHelperFactory indexHelperFactory,
boolean retainInput, boolean retainMissing, IMissingWriterFactory
missingWriterFactory,
ISearchOperationCallbackFactory searchCallbackFactory, boolean
appendIndexFilter,
- boolean appendSearchCallbackProceedResult, byte[]
searchCallbackProceedResultFalseValue,
- byte[] searchCallbackProceedResultTrueValue) throws
HyracksDataException {
+ ITupleFilterFactory tupleFilterFactory, long outputLimit) throws
HyracksDataException {
+ this(ctx, inputRecDesc, partition, minFilterFieldIndexes,
maxFilterFieldIndexes, indexHelperFactory,
+ retainInput, retainMissing, missingWriterFactory,
searchCallbackFactory, appendIndexFilter,
+ tupleFilterFactory, outputLimit, false, null, null);
+ }
+
+ public IndexSearchOperatorNodePushable(IHyracksTaskContext ctx,
RecordDescriptor inputRecDesc, int partition,
+ int[] minFilterFieldIndexes, int[] maxFilterFieldIndexes,
IIndexDataflowHelperFactory indexHelperFactory,
+ boolean retainInput, boolean retainMissing, IMissingWriterFactory
missingWriterFactory,
+ ISearchOperationCallbackFactory searchCallbackFactory, boolean
appendIndexFilter,
+ ITupleFilterFactory tupleFactoryFactory, long outputLimit, boolean
appendSearchCallbackProceedResult,
+ byte[] searchCallbackProceedResultFalseValue, byte[]
searchCallbackProceedResultTrueValue)
+ throws HyracksDataException {
this.ctx = ctx;
this.indexHelper =
indexHelperFactory.create(ctx.getJobletContext().getServiceContext(),
partition);
this.retainInput = retainInput;
@@ -138,6 +161,8 @@
if (ctx.getStatsCollector() != null) {
ctx.getStatsCollector().add(stats);
}
+ this.tupleFilterFactory = tupleFactoryFactory;
+ this.outputLimit = outputLimit;
}
protected abstract ISearchPredicate createSearchPredicate();
@@ -179,6 +204,12 @@
buildMissingTuple(numIndexFilterFields, nonFilterTupleBuild,
nonMatchWriter);
}
+ if (tupleFilterFactory != null) {
+ tupleFilter = tupleFilterFactory.createTupleFilter(ctx);
+ referenceFilterTuple = new ReferenceFrameTupleReference();
+ }
+ finished = false;
+
try {
searchPred = createSearchPredicate();
tb = new ArrayTupleBuilder(recordDesc.getFieldCount());
@@ -201,9 +232,13 @@
protected void writeSearchResults(int tupleIndex) throws Exception {
long matchingTupleCount = 0;
while (cursor.hasNext()) {
- matchingTupleCount++;
- tb.reset();
cursor.next();
+ if (tupleFilter != null &&
!tupleFilter.accept(referenceFilterTuple.reset(cursor.getTuple()))) {
+ continue;
+ }
+ tb.reset();
+ matchingTupleCount++;
+
if (retainInput) {
frameTuple.reset(accessor, tupleIndex);
for (int i = 0; i < frameTuple.getFieldCount(); i++) {
@@ -222,6 +257,10 @@
writeFilterTupleToOutput(((ILSMIndexCursor)
cursor).getFilterMaxTuple());
}
FrameUtils.appendToWriter(writer, appender,
tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
+ if (outputLimit >= 0 && ++outputCount >= outputLimit) {
+ finished = true;
+ break;
+ }
}
stats.getTupleCounter().update(matchingTupleCount);
@@ -237,7 +276,7 @@
accessor.reset(buffer);
int tupleCount = accessor.getTupleCount();
try {
- for (int i = 0; i < tupleCount; i++) {
+ for (int i = 0; i < tupleCount && !finished; i++) {
resetSearchPredicate(i);
cursor.close();
indexAccessor.search(cursor, searchPred);
@@ -344,4 +383,45 @@
}
}
+ private static class ReferenceFrameTupleReference implements
IFrameTupleReference {
+ private ITupleReference tuple;
+
+ public IFrameTupleReference reset(ITupleReference tuple) {
+ this.tuple = tuple;
+ return this;
+ }
+
+ @Override
+ public int getFieldCount() {
+ return tuple.getFieldCount();
+ }
+
+ @Override
+ public byte[] getFieldData(int fIdx) {
+ return tuple.getFieldData(fIdx);
+ }
+
+ @Override
+ public int getFieldStart(int fIdx) {
+ return tuple.getFieldStart(fIdx);
+ }
+
+ @Override
+ public int getFieldLength(int fIdx) {
+ return tuple.getFieldLength(fIdx);
+ }
+
+ @Override
+ public IFrameTupleAccessor getFrameTupleAccessor() {
+ throw new UnsupportedOperationException(
+ "getFrameTupleAccessor is not supported by
ReferenceFrameTupleReference");
+ }
+
+ @Override
+ public int getTupleIndex() {
+ throw new UnsupportedOperationException("getTupleIndex is not
supported by ReferenceFrameTupleReference");
+ }
+
+ }
+
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorNodePushable.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorNodePushable.java
index 18ced6d..4dcfa94 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorNodePushable.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorNodePushable.java
@@ -55,8 +55,9 @@
boolean appendIndexFilter, boolean appendOpCallbackProceedResult,
byte[] searchCallbackProceedResultFalseValue, byte[]
searchCallbackProceedResultTrueValue)
throws HyracksDataException {
+ // TODO: predicate & limit pushdown not enabled for RTree yet
super(ctx, inputRecDesc, partition, minFilterFieldIndexes,
maxFilterFieldIndexes, indexHelperFactory,
- retainInput, retainMissing, missingWriterFactory,
searchCallbackFactory, appendIndexFilter,
+ retainInput, retainMissing, missingWriterFactory,
searchCallbackFactory, appendIndexFilter, null, -1,
appendOpCallbackProceedResult,
searchCallbackProceedResultFalseValue,
searchCallbackProceedResultTrueValue);
if (keyFields != null && keyFields.length > 0) {
--
To view, visit https://asterix-gerrit.ics.uci.edu/2541
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I824fcad79995325e12a1a81d629160025294b915
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Luo Chen <[email protected]>
