Taewoo Kim has submitted this change and it was merged. Change subject: [ASTERIXDB-2374][RT] Index-only plan on B+Tree disk components ......................................................................
[ASTERIXDB-2374][RT] Index-only plan on B+Tree disk components - user model changes: no - storage format changes: no - interface changes: no Details: Let the index-only plan properly work on the disk components of B+Tree. Currently, only the records from in-memory components has been applied because searchCallback.proceed() is only called for those. Change-Id: I655eacc5517352a382d1b61f7b630f0f307b7c7b Reviewed-on: https://asterix-gerrit.ics.uci.edu/2623 Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Contrib: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Reviewed-by: Dmitry Lychagin <dmitry.lycha...@couchbase.com> --- M asterixdb/asterix-app/src/test/resources/runtimets/metrics.xml A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/secondary-index-index-only/secondary-index-index-only.1.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/secondary-index-index-only/secondary-index-index-only.2.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/secondary-index-index-only/secondary-index-index-only.3.metrics.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/secondary-index-index-only/secondary-index-index-only.4.ddl.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/secondary-index/secondary-index.3.metrics.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/results/metrics/secondary-index-index-only/secondary-index-index-only.3.regexadm M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeRangeSearchCursor.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeSearchCursor.java 9 files changed, 169 insertions(+), 41 deletions(-) Approvals: Anon. E. Moose #1000171: Jenkins: Verified; ; Verified Dmitry Lychagin: Looks good to me, approved Objections: Jenkins: Violations found diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/metrics.xml b/asterixdb/asterix-app/src/test/resources/runtimets/metrics.xml index f65ede3..da7ba31 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/metrics.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/metrics.xml @@ -35,5 +35,10 @@ <output-dir compare="Text">secondary-index</output-dir> </compilation-unit> </test-case> + <test-case FilePath="metrics"> + <compilation-unit name="secondary-index-index-only"> + <output-dir compare="Text">secondary-index-index-only</output-dir> + </compilation-unit> + </test-case> </test-group> </test-suite> \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/secondary-index-index-only/secondary-index-index-only.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/secondary-index-index-only/secondary-index-index-only.1.ddl.sqlpp new file mode 100644 index 0000000..c4d77ca --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/secondary-index-index-only/secondary-index-index-only.1.ddl.sqlpp @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Description : Processed objects metrics on secondary index scan when it's an index-only plan + * Expected Res : Success + * Date : 27 Apr 2018 + */ + +drop dataverse test if exists; +create dataverse test; + +use test; + +create type test.AddressType as +{ + number : bigint, + street : string, + city : string +}; + +create type test.CustomerType as + closed { + cid : bigint, + name : string, + age : bigint?, + address : AddressType?, + lastorder : { + oid : bigint, + total : float + } +}; + +create dataset Customers(CustomerType) primary key cid; +create index customer_name_idx on Customers(name); \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/secondary-index-index-only/secondary-index-index-only.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/secondary-index-index-only/secondary-index-index-only.2.update.sqlpp new file mode 100644 index 0000000..6a5fd9e --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/secondary-index-index-only/secondary-index-index-only.2.update.sqlpp @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use test; + +load dataset Customers using localfs + ((`path`=`asterix_nc1://data/custord-tiny/customer-tiny-neg.adm`), + (`format`=`adm`)); \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/secondary-index-index-only/secondary-index-index-only.3.metrics.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/secondary-index-index-only/secondary-index-index-only.3.metrics.sqlpp new file mode 100644 index 0000000..881ed24 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/secondary-index-index-only/secondary-index-index-only.3.metrics.sqlpp @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use test; + +select count(*) from Customers +where name = "Marvella Loud"; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/secondary-index-index-only/secondary-index-index-only.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/secondary-index-index-only/secondary-index-index-only.4.ddl.sqlpp new file mode 100644 index 0000000..f12a2b7 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/secondary-index-index-only/secondary-index-index-only.4.ddl.sqlpp @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +drop dataverse test; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/secondary-index/secondary-index.3.metrics.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/secondary-index/secondary-index.3.metrics.sqlpp index 1c5126f..95d4681 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/secondary-index/secondary-index.3.metrics.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/secondary-index/secondary-index.3.metrics.sqlpp @@ -24,5 +24,7 @@ use test; +set noindexonly 'true'; + select count(*) from Customers where name = "Marvella Loud"; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/metrics/secondary-index-index-only/secondary-index-index-only.3.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/metrics/secondary-index-index-only/secondary-index-index-only.3.regexadm new file mode 100644 index 0000000..91a2dfd --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/metrics/secondary-index-index-only/secondary-index-index-only.3.regexadm @@ -0,0 +1 @@ +.*"processedObjects":1.* \ No newline at end of file diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeRangeSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeRangeSearchCursor.java index a675047..6eef5a9 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeRangeSearchCursor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeRangeSearchCursor.java @@ -105,48 +105,50 @@ while (!outputPriorityQueue.isEmpty() || needPushElementIntoQueue) { if (!outputPriorityQueue.isEmpty()) { PriorityQueueElement queueHead = outputPriorityQueue.peek(); - if (canCallProceed && includeMutableComponent) { + if (canCallProceed) { resultOfSearchCallbackProceed = searchCallback.proceed(queueHead.getTuple()); - if (!resultOfSearchCallbackProceed) { - // In case proceed() fails and there is an in-memory component, - // we can't simply use this element since there might be a change. - PriorityQueueElement mutableElement = remove(outputPriorityQueue, 0); - if (mutableElement != null) { - // Copies the current queue head - if (tupleBuilder == null) { - tupleBuilder = new ArrayTupleBuilder(cmp.getKeyFieldCount()); + if (includeMutableComponent) { + if (!resultOfSearchCallbackProceed) { + // In case proceed() fails and there is an in-memory component, + // we can't simply use this element since there might be a change. + PriorityQueueElement mutableElement = remove(outputPriorityQueue, 0); + if (mutableElement != null) { + // Copies the current queue head + if (tupleBuilder == null) { + tupleBuilder = new ArrayTupleBuilder(cmp.getKeyFieldCount()); + } + TupleUtils.copyTuple(tupleBuilder, queueHead.getTuple(), cmp.getKeyFieldCount()); + copyTuple.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray()); + // Unlatches/unpins the leaf page of the index. + rangeCursors[0].close(); + // Reconcile. + searchCallback.reconcile(copyTuple); + // Re-traverses the index. + reusablePred.setLowKey(copyTuple, true); + btreeAccessors[0].search(rangeCursors[0], reusablePred); + pushIntoQueueFromCursorAndReplaceThisElement(mutableElement); + // now that we have completed the search and we have latches over the pages, + // it is safe to complete the operation.. but as per the API of the callback + // we only complete if we're producing this tuple + // get head again + queueHead = outputPriorityQueue.peek(); + /* + * We need to restart in one of two cases: + * 1. no more elements in the priority queue. + * 2. the key of the head has changed (which means we need to call proceed) + */ + if (queueHead == null || cmp.compare(copyTuple, queueHead.getTuple()) != 0) { + // cancel since we're not continuing + searchCallback.cancel(copyTuple); + continue; + } + searchCallback.complete(copyTuple); + // it is safe to proceed now + } else { + // There are no more elements in the memory component.. can safely skip locking for the + // remaining operations + includeMutableComponent = false; } - TupleUtils.copyTuple(tupleBuilder, queueHead.getTuple(), cmp.getKeyFieldCount()); - copyTuple.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray()); - // Unlatches/unpins the leaf page of the index. - rangeCursors[0].close(); - // Reconcile. - searchCallback.reconcile(copyTuple); - // Re-traverses the index. - reusablePred.setLowKey(copyTuple, true); - btreeAccessors[0].search(rangeCursors[0], reusablePred); - pushIntoQueueFromCursorAndReplaceThisElement(mutableElement); - // now that we have completed the search and we have latches over the pages, - // it is safe to complete the operation.. but as per the API of the callback - // we only complete if we're producing this tuple - // get head again - queueHead = outputPriorityQueue.peek(); - /* - * We need to restart in one of two cases: - * 1. no more elements in the priority queue. - * 2. the key of the head has changed (which means we need to call proceed) - */ - if (queueHead == null || cmp.compare(copyTuple, queueHead.getTuple()) != 0) { - // cancel since we're not continuing - searchCallback.cancel(copyTuple); - continue; - } - searchCallback.complete(copyTuple); - // it is safe to proceed now - } else { - // There are no more elements in the memory component.. can safely skip locking for the - // remaining operations - includeMutableComponent = false; } } } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeSearchCursor.java index baf0d4a..efacad1 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeSearchCursor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeSearchCursor.java @@ -107,6 +107,6 @@ @Override public boolean getSearchOperationCallbackProceedResult() { - return false; + return currentCursor.getSearchOperationCallbackProceedResult(); } } -- To view, visit https://asterix-gerrit.ics.uci.edu/2623 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: I655eacc5517352a382d1b61f7b630f0f307b7c7b Gerrit-PatchSet: 3 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Taewoo Kim <wangs...@gmail.com> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Dmitry Lychagin <dmitry.lycha...@couchbase.com> Gerrit-Reviewer: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Gerrit-Reviewer: Taewoo Kim <wangs...@gmail.com>
