Taewoo Kim has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/2854
Change subject: [ASTERIXDB-2437][COMP] Ensure the index-only plan on a
composite index
......................................................................
[ASTERIXDB-2437][COMP] Ensure the index-only plan on a composite index
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Ensure the proper build of an index-only plan on a composite index
where both fields are used in the SELECT condition and only one field
is returned.
Change-Id: Idcc4cbe08323e0c6edb4a01637b2017128da1ab5
---
M
asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
A
asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-sidx-idxonly-10.sqlpp
A
asterixdb/asterix-app/src/test/resources/optimizerts/results/btree-index/btree-sidx-idxonly-10.plan
3 files changed, 177 insertions(+), 11 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/54/2854/1
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
index c925b55..b1de3a1 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
@@ -1053,7 +1053,7 @@
constAssignOp.setSourceLocation(sourceLoc);
if (constantAssignVarUsedInTopOp) {
// Places this assign after the secondary index-search op.
- constAssignOp.getInputs().add(new
MutableObject<ILogicalOperator>(inputOp));
+ constAssignOp.getInputs().add(new
MutableObject<>(inputOp));
constAssignOp.setExecutionMode(ExecutionMode.PARTITIONED);
context.computeAndSetTypeEnvironmentForOperator(constAssignOp);
currentOp = constAssignOp;
@@ -1082,7 +1082,7 @@
splitOp = new SplitOperator(2,
new MutableObject<ILogicalExpression>(new
VariableReferenceExpression(condSplitVars.get(0))));
splitOp.setSourceLocation(sourceLoc);
- splitOp.getInputs().add(new
MutableObject<ILogicalOperator>(currentOp));
+ splitOp.getInputs().add(new MutableObject<>(currentOp));
splitOp.setExecutionMode(ExecutionMode.PARTITIONED);
context.computeAndSetTypeEnvironmentForOperator(splitOp);
@@ -1189,11 +1189,40 @@
LogicalVariable replacedVar = context.newVar();
origPKRecAndSKVarToleftPathMap.put(tVar, replacedVar);
origVarToOutputVarMap.put(skVarsFromSIdxUnnestMap.get(sIndexIdx),
tVar);
- unionVarMap.add(new Triple<LogicalVariable, LogicalVariable,
LogicalVariable>(replacedVar,
+ unionVarMap.add(new Triple<>(replacedVar,
skVarsFromSIdxUnnestMap.get(sIndexIdx), tVar));
// Constructs the mapping between the SK from the original
data-scan
// and the SK from the secondary index search since they are
different logical variables.
origVarToSIdxUnnestMapOpVarMap.put(tVar,
skVarsFromSIdxUnnestMap.get(sIndexIdx));
+ }
+
+ // For B-Tree case: if the given secondary key field variable is used
only in the select or
+ // join condition, we were not able to catch the mapping between the
the SK from the original
+ // data-scan and the SK from the secondary index search since they are
different logical variables.
+ // (E.g., we are sending a query on a composite index but returns only
one field.)
+ List<LogicalVariable> varsUsedInTopOpButNotAfterwards = new
ArrayList<>();
+ copyVarsToAnotherList(uniqueUsedVarsInTopOp,
varsUsedInTopOpButNotAfterwards);
+ for (LogicalVariable v : usedVarsAfterTopOp) {
+ if (varsUsedInTopOpButNotAfterwards.contains(v)) {
+ varsUsedInTopOpButNotAfterwards.remove(v);
+ }
+ }
+ if (idxType == IndexType.BTREE) {
+ for (LogicalVariable v : varsUsedInTopOpButNotAfterwards) {
+ int sIndexIdx =
chosenIndexFieldNames.indexOf(subTree.getVarsToFieldNameMap().get(v));
+ // For the join-case, the match might not exist.
+ // In this case, we just propagate the variables later.
+ if (sIndexIdx == -1) {
+ continue;
+ }
+ // B-Tree case:
+ LogicalVariable replacedVar = context.newVar();
+ origPKRecAndSKVarToleftPathMap.put(v, replacedVar);
+
origVarToOutputVarMap.put(skVarsFromSIdxUnnestMap.get(sIndexIdx), v);
+ // Constructs the mapping between the SK from the original
data-scan
+ // and the SK from the secondary index search since they are
different logical variables.
+ origVarToSIdxUnnestMapOpVarMap.put(v,
skVarsFromSIdxUnnestMap.get(sIndexIdx));
+ }
}
// For R-Tree case: if the given secondary key field variable is used
only in the select or join condition,
@@ -1248,7 +1277,7 @@
ILogicalExpression conditionRefExpr =
conditionRef.getValue().cloneExpression();
// The retainMissing variable contains the information whether we are
optimizing a left-outer join or not.
LogicalVariable newMissingPlaceHolderVar = retainMissing ?
newMissingPlaceHolderForLOJ : null;
- newSelectOpInLeftPath = new SelectOperator(new
MutableObject<ILogicalExpression>(conditionRefExpr),
+ newSelectOpInLeftPath = new SelectOperator(new
MutableObject<>(conditionRefExpr),
retainMissing, newMissingPlaceHolderVar);
newSelectOpInLeftPath.setSourceLocation(conditionRefExpr.getSourceLocation());
VariableUtilities.substituteVariables(newSelectOpInLeftPath,
origVarToNewVarInLeftPathMap, context);
@@ -1273,7 +1302,7 @@
}
newSelectOpInLeftPath.getInputs().clear();
newSelectOpInLeftPath.getInputs()
- .add(new
MutableObject<ILogicalOperator>(assignsBeforeTopOpRef.get(0).getValue()));
+ .add(new
MutableObject<>(assignsBeforeTopOpRef.get(0).getValue()));
} else {
newSelectOpInLeftPath.getInputs().add(new
MutableObject<ILogicalOperator>(primaryIndexUnnestMapOp));
}
@@ -1308,10 +1337,10 @@
// since we need to change the variable reference in the SELECT
operator.
// For the index-nested-loop join case, we copy the condition of
the join operator.
ILogicalExpression conditionRefExpr2 =
conditionRef.getValue().cloneExpression();
- newSelectOpInRightPath = new SelectOperator(new
MutableObject<ILogicalExpression>(conditionRefExpr2),
+ newSelectOpInRightPath = new SelectOperator(new
MutableObject<>(conditionRefExpr2),
retainMissing, newMissingPlaceHolderVar);
newSelectOpInRightPath.setSourceLocation(conditionRefExpr2.getSourceLocation());
- newSelectOpInRightPath.getInputs().add(new
MutableObject<ILogicalOperator>(currentTopOpInRightPath));
+ newSelectOpInRightPath.getInputs().add(new
MutableObject<>(currentTopOpInRightPath));
VariableUtilities.substituteVariables(newSelectOpInRightPath,
origVarToSIdxUnnestMapOpVarMap, context);
VariableUtilities.substituteVariables(newSelectOpInRightPath,
origSKFieldVarToNewSKFieldVarMap, context);
newSelectOpInRightPath.setExecutionMode(ExecutionMode.PARTITIONED);
@@ -1331,7 +1360,7 @@
unionAllOp = new UnionAllOperator(unionVarMap);
unionAllOp.setSourceLocation(sourceLoc);
unionAllOp.getInputs().add(new
MutableObject<ILogicalOperator>(newSelectOpInLeftPath));
- unionAllOp.getInputs().add(new
MutableObject<ILogicalOperator>(currentTopOpInRightPath));
+ unionAllOp.getInputs().add(new
MutableObject<>(currentTopOpInRightPath));
unionAllOp.setExecutionMode(ExecutionMode.PARTITIONED);
context.computeAndSetTypeEnvironmentForOperator(unionAllOp);
@@ -1429,7 +1458,7 @@
// Common part for the non-index-only plan and index-only plan
// Variables and types for the primary-index search.
List<LogicalVariable> primaryIndexUnnestVars = new ArrayList<>();
- List<Object> primaryIndexOutputTypes = new ArrayList<Object>();
+ List<Object> primaryIndexOutputTypes = new ArrayList<>();
// Appends output variables/types generated by the primary-index
search (not forwarded from input).
primaryIndexUnnestVars.addAll(dataSourceOp.getVariables());
appendPrimaryIndexTypes(dataset, recordType, metaRecordType,
primaryIndexOutputTypes);
@@ -2479,9 +2508,9 @@
if (LOJVarExist && LOJVarCanBeDeleted) {
UnionAllOperator newUnionAllOp = new UnionAllOperator(varMap);
newUnionAllOp.getInputs()
- .add(new
MutableObject<ILogicalOperator>(unionAllOp.getInputs().get(0).getValue()));
+ .add(new
MutableObject<>(unionAllOp.getInputs().get(0).getValue()));
newUnionAllOp.getInputs()
- .add(new
MutableObject<ILogicalOperator>(unionAllOp.getInputs().get(1).getValue()));
+ .add(new
MutableObject<>(unionAllOp.getInputs().get(1).getValue()));
context.computeAndSetTypeEnvironmentForOperator(newUnionAllOp);
return newUnionAllOp;
} else {
diff --git
a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-sidx-idxonly-10.sqlpp
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-sidx-idxonly-10.sqlpp
new file mode 100644
index 0000000..6242a27
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-sidx-idxonly-10.sqlpp
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description : Secondary BTree Index index-only selection plan
verification test
+ * : The test is intended to verify that the secondary BTree
index is used in the optimized query plan.
+ * : In this plan, we fetch PK and SK based on a select
condition that utilizes a secondary index.
+ * : The plan should have two paths after the secondary
index-lookup.
+ * : The left path:
+ * ... -> unnest-map (sidx) -> split -> unnest-map (pidx)
-> select -> union -> ...
+ * : The right path:
+ * ... -> unnest-map (sidx) -> split ->
-> union -> ...
+ * Expected Result : Success
+ *
+*/
+
+drop dataverse twitter if exists;
+create dataverse twitter if not exists;
+use twitter;
+
+create type typeUser if not exists as open {
+ id: int64,
+ name: string,
+ screen_name : string,
+ profile_image_url : string,
+ lang : string,
+ location: string,
+ create_at: date,
+ description: string,
+ followers_count: int32,
+ friends_count: int32,
+ statues_count: int64
+};
+
+create type typePlace if not exists as open{
+ country : string,
+ country_code : string,
+ full_name : string,
+ id : string,
+ name : string,
+ place_type : string,
+ bounding_box : rectangle
+};
+
+create type typeGeoTag if not exists as open {
+ stateID: int32,
+ stateName: string,
+ countyID: int32,
+ countyName: string,
+ cityID: int32?,
+ cityName: string?
+};
+
+create type typeTweet if not exists as open {
+ create_at : datetime,
+ id: int64,
+ text: string,
+ in_reply_to_status : int64,
+ in_reply_to_user : int64,
+ favorite_count : int64,
+ coordinate: point?,
+ retweet_count : int64,
+ lang : string,
+ is_retweet: boolean,
+ hashtags : {{ string }} ?,
+ user_mentions : {{ int64 }} ? ,
+ user : typeUser,
+ place : typePlace?,
+ geo_tag: typeGeoTag
+};
+
+create dataset ds_tweet(typeTweet) if not exists primary key id;
+
+create index create_at_status_count_idx on ds_tweet(user.create_at,
user.statues_count);
+
+select value count(first.create_at) from (
+select t.user.create_at, t.user.statues_count, t.id from ds_tweet t
+where
+ t.user.create_at >=
+ date_from_unix_time_in_days(10000) and
+ t.user.create_at <
+ date_from_unix_time_in_days(12000) and
+ t.user.statues_count >= 0 and
+ t.user.statues_count < 1000000
+) first;
+
diff --git
a/asterixdb/asterix-app/src/test/resources/optimizerts/results/btree-index/btree-sidx-idxonly-10.plan
b/asterixdb/asterix-app/src/test/resources/optimizerts/results/btree-index/btree-sidx-idxonly-10.plan
new file mode 100644
index 0000000..89e06cc
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/optimizerts/results/btree-index/btree-sidx-idxonly-10.plan
@@ -0,0 +1,35 @@
+-- DISTRIBUTE_RESULT |UNPARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED|
+ -- AGGREGATE |UNPARTITIONED|
+ -- RANDOM_MERGE_EXCHANGE |PARTITIONED|
+ -- AGGREGATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- UNION_ALL |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- SPLIT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE
|PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- SPLIT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
--
To view, visit https://asterix-gerrit.ics.uci.edu/2854
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Idcc4cbe08323e0c6edb4a01637b2017128da1ab5
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Taewoo Kim <[email protected]>
