Luo Chen has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/2531
Change subject: [ASTERIXDB-2334] Fix Range Predicate for Composite Key Search
......................................................................
[ASTERIXDB-2334] Fix Range Predicate for Composite Key Search
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Currently during a primary btree search, we always use the range limit
of the first search key. However, this is incorrect for prefix range
search. This patch fixes this bug by using the correct range limit
during index search.
Change-Id: I6dbbc1b7b78762737a21773efc53035fc62d8f24
---
M
asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
M
asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
M
asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/composite-key/composite-prefix/composite-prefix.1.ddl.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/composite-key/composite-prefix/composite-prefix.2.update.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/composite-key/composite-prefix/composite-prefix.3.query.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/composite-key/query-ASTERIXDB-2334/query-ASTERIXDB-2334.1.ddl.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/composite-key/query-ASTERIXDB-2334/query-ASTERIXDB-2334.2.update.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/composite-key/query-ASTERIXDB-2334/query-ASTERIXDB-920.3.query.sqlpp
M
asterixdb/asterix-app/src/test/resources/runtimets/results/composite-key/composite-low-high/composite-low-high.1.adm
A
asterixdb/asterix-app/src/test/resources/runtimets/results/composite-key/composite-prefix/composite-prefix.1.adm
A
asterixdb/asterix-app/src/test/resources/runtimets/results/composite-key/query-ASTERIXDB-2334/query-ASTERIXDB-2334.1.adm
M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
M
hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/ConcatenatingTupleReference.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDeletedKeysBTreeMergeCursor.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListCursor.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListScanCursor.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexOpContext.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
M
hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
28 files changed, 373 insertions(+), 139 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/31/2531/1
diff --git
a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
index bb76204..2aebc59 100644
---
a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
+++
b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
@@ -178,7 +178,7 @@
private void stopIfRunning(ActiveRuntimeId runtimeId, IActiveRuntime
runtime)
throws HyracksDataException, InterruptedException {
- if (runtimes.remove(runtimeId) != null) {
+ if (runtimes.containsKey(runtimeId)) {
runtime.stop();
} else {
LOGGER.info("Not stopping already stopped runtime " + runtimeId);
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
index a82e780..60d4d3d 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
@@ -528,11 +528,6 @@
}
}
- if (primaryIndexPostProccessingIsNeeded) {
- Arrays.fill(lowKeyInclusive, true);
- Arrays.fill(highKeyInclusive, true);
- }
-
// determine cases when prefix search could be applied
for (int i = 1; i < lowKeyExprs.length; i++) {
if (lowKeyLimits[0] == null && lowKeyLimits[i] != null ||
lowKeyLimits[0] != null && lowKeyLimits[i] == null
@@ -542,6 +537,12 @@
primaryIndexPostProccessingIsNeeded = true;
}
}
+
+ if (primaryIndexPostProccessingIsNeeded) {
+ Arrays.fill(lowKeyInclusive, true);
+ Arrays.fill(highKeyInclusive, true);
+ }
+
if (lowKeyLimits[0] == null) {
lowKeyInclusive[0] = true;
}
@@ -563,8 +564,10 @@
BTreeJobGenParams jobGenParams = new
BTreeJobGenParams(chosenIndex.getIndexName(), IndexType.BTREE,
dataset.getDataverseName(), dataset.getDatasetName(),
retainInput, requiresBroadcast);
- jobGenParams.setLowKeyInclusive(lowKeyInclusive[0]);
- jobGenParams.setHighKeyInclusive(highKeyInclusive[0]);
+ jobGenParams
+
.setLowKeyInclusive(lowKeyInclusive[primaryIndexPostProccessingIsNeeded ? 0 :
numSecondaryKeys - 1]);
+ jobGenParams
+
.setHighKeyInclusive(highKeyInclusive[primaryIndexPostProccessingIsNeeded ? 0 :
numSecondaryKeys - 1]);
jobGenParams.setIsEqCondition(isEqCondition);
jobGenParams.setLowKeyVarList(keyVarList, 0, numLowKeys);
jobGenParams.setHighKeyVarList(keyVarList, numLowKeys, numHighKeys);
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
index acc3970..cc70d57 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
@@ -30,6 +30,7 @@
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.List;
+import java.util.concurrent.TimeUnit;
import java.util.function.BiPredicate;
import java.util.stream.Stream;
@@ -70,6 +71,7 @@
joinPath(getProjectPath().toString(), "src", "test", "resources",
"cc.conf");
private static final String DEFAULT_STORAGE_PATH = joinPath("target",
"io", "dir");
private static String storagePath = DEFAULT_STORAGE_PATH;
+ private static final long RESULT_TTL = TimeUnit.MINUTES.toMillis(5);
static {
System.setProperty("java.util.logging.manager",
org.apache.logging.log4j.jul.LogManager.class.getName());
@@ -198,7 +200,7 @@
ccConfig.setClientListenAddress(Inet4Address.getLoopbackAddress().getHostAddress());
ccConfig.setClientListenPort(DEFAULT_HYRACKS_CC_CLIENT_PORT);
ccConfig.setClusterListenPort(DEFAULT_HYRACKS_CC_CLUSTER_PORT);
- ccConfig.setResultTTL(120000L);
+ ccConfig.setResultTTL(RESULT_TTL);
ccConfig.setResultSweepThreshold(1000L);
ccConfig.setEnforceFrameWriterProtocol(true);
configManager.set(ControllerConfig.Option.DEFAULT_DIR,
joinPath(getDefaultStoragePath(), "asterixdb"));
@@ -217,7 +219,7 @@
ncConfig.setDataListenAddress(Inet4Address.getLoopbackAddress().getHostAddress());
ncConfig.setResultListenAddress(Inet4Address.getLoopbackAddress().getHostAddress());
ncConfig.setMessagingListenAddress(Inet4Address.getLoopbackAddress().getHostAddress());
- ncConfig.setResultTTL(120000L);
+ ncConfig.setResultTTL(RESULT_TTL);
ncConfig.setResultSweepThreshold(1000L);
ncConfig.setVirtualNC();
configManager.set(ControllerConfig.Option.DEFAULT_DIR,
joinPath(getDefaultStoragePath(), "asterixdb", ncName));
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/composite-key/composite-prefix/composite-prefix.1.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/composite-key/composite-prefix/composite-prefix.1.ddl.sqlpp
new file mode 100644
index 0000000..ad1c308
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/composite-key/composite-prefix/composite-prefix.1.ddl.sqlpp
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : This test case is to verify composite key prefix search
+ * Expected Res : Success
+ * Date : 23 March 2018
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+create type PointType as closed {
+x:int,
+y:int
+};
+
+create dataset Points(PointType) primary key x, y;
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/composite-key/composite-prefix/composite-prefix.2.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/composite-key/composite-prefix/composite-prefix.2.update.sqlpp
new file mode 100644
index 0000000..122b5a8
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/composite-key/composite-prefix/composite-prefix.2.update.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+insert into Points {"x": 1, "y": 1};
+insert into Points {"x": 2, "y": 1};
+insert into Points {"x": 2, "y": 2};
+insert into Points {"x": 2, "y": 3};
+insert into Points {"x": 2, "y": 4};
+insert into Points {"x": 2, "y": 5};
+insert into Points {"x": 3, "y": 1};
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/composite-key/composite-prefix/composite-prefix.3.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/composite-key/composite-prefix/composite-prefix.3.query.sqlpp
new file mode 100644
index 0000000..c37d492
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/composite-key/composite-prefix/composite-prefix.3.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+use test;
+
+select value p
+from Points p
+where p.x=2
+and p.y>=2 and p.y<4
+order by p.x,p.y;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/composite-key/query-ASTERIXDB-2334/query-ASTERIXDB-2334.1.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/composite-key/query-ASTERIXDB-2334/query-ASTERIXDB-2334.1.ddl.sqlpp
new file mode 100644
index 0000000..0e7bad9
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/composite-key/query-ASTERIXDB-2334/query-ASTERIXDB-2334.1.ddl.sqlpp
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : This test case is to verify the fix for ASTERIXDB-2334
+ : https://issues.apache.org/jira/browse/ASTERIXDB-2334
+ * Expected Res : Success
+ * Date : 23 March 2018
+ */
+
+DROP DATAVERSE earthquake IF EXISTS;
+CREATE DATAVERSE earthquake;
+USE earthquake;
+
+CREATE TYPE QzExternalTypeNew AS {
+stationid: string,
+pointid: string,
+itemid: string,
+samplerate: string,
+startdate: string,
+obsvalue: string
+};
+
+CREATE DATASET qz9130all(QzExternalTypeNew) PRIMARY KEY
stationid,pointid,itemid,samplerate,startdate;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/composite-key/query-ASTERIXDB-2334/query-ASTERIXDB-2334.2.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/composite-key/query-ASTERIXDB-2334/query-ASTERIXDB-2334.2.update.sqlpp
new file mode 100644
index 0000000..e2d6046
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/composite-key/query-ASTERIXDB-2334/query-ASTERIXDB-2334.2.update.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use earthquake;
+
+INSERT INTO qz9130all(
{"stationid":"01","pointid":"5","itemid":"9130","samplerate":"01","startdate":"20080509","obsvalue":"9"}
);
+INSERT INTO qz9130all(
{"stationid":"01","pointid":"5","itemid":"9130","samplerate":"01","startdate":"20080510","obsvalue":"9"}
);
+INSERT INTO qz9130all(
{"stationid":"01","pointid":"5","itemid":"9130","samplerate":"01","startdate":"20080511","obsvalue":"9"}
);
+INSERT INTO qz9130all(
{"stationid":"01","pointid":"5","itemid":"9130","samplerate":"01","startdate":"20080512","obsvalue":"9"}
);
+INSERT INTO qz9130all(
{"stationid":"01","pointid":"5","itemid":"9130","samplerate":"01","startdate":"20080513","obsvalue":"9"}
);
+INSERT INTO qz9130all(
{"stationid":"01","pointid":"5","itemid":"9130","samplerate":"01","startdate":"20080514","obsvalue":"9"}
);
+INSERT INTO qz9130all(
{"stationid":"01","pointid":"5","itemid":"9130","samplerate":"01","startdate":"20080515","obsvalue":"9"}
);
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/composite-key/query-ASTERIXDB-2334/query-ASTERIXDB-920.3.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/composite-key/query-ASTERIXDB-2334/query-ASTERIXDB-920.3.query.sqlpp
new file mode 100644
index 0000000..622937e
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/composite-key/query-ASTERIXDB-2334/query-ASTERIXDB-920.3.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+use earthquake;
+
+SELECT startdate
+FROM qz9130all
+WHERE samplerate='01' and stationid='01' and pointid='5' and itemid='9130' and
startdate >= '20080510' and startdate < '20080513'
+ORDER BY startdate;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/composite-key/composite-low-high/composite-low-high.1.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/composite-key/composite-low-high/composite-low-high.1.adm
index 8c4166d..625f0fd 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/composite-key/composite-low-high/composite-low-high.1.adm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/composite-key/composite-low-high/composite-low-high.1.adm
@@ -1,3 +1,3 @@
{ "x": 15, "y": 40 }
{ "x": 20, "y": 50 }
-{ "x": 60, "y": 40 }
\ No newline at end of file
+{ "x": 60, "y": 40 }
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/composite-key/composite-prefix/composite-prefix.1.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/composite-key/composite-prefix/composite-prefix.1.adm
new file mode 100644
index 0000000..dbf0ba7
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/composite-key/composite-prefix/composite-prefix.1.adm
@@ -0,0 +1,2 @@
+{ "x": 2, "y": 2 }
+{ "x": 2, "y": 3 }
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/composite-key/query-ASTERIXDB-2334/query-ASTERIXDB-2334.1.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/composite-key/query-ASTERIXDB-2334/query-ASTERIXDB-2334.1.adm
new file mode 100644
index 0000000..41da74e
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/composite-key/query-ASTERIXDB-2334/query-ASTERIXDB-2334.1.adm
@@ -0,0 +1,3 @@
+{ "startdate": "20080510" }
+{ "startdate": "20080511" }
+{ "startdate": "20080512" }
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 0967ba6..832bb06 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -9743,10 +9743,20 @@
<output-dir compare="Text">query-ASTERIXDB-920</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="composite-key">
+ <compilation-unit name="query-ASTERIXDB-2334">
+ <output-dir compare="Text">query-ASTERIXDB-2334</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="composite-key">
<compilation-unit name="composite-low-high">
<output-dir compare="Text">composite-low-high</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="composite-key">
+ <compilation-unit name="composite-prefix">
+ <output-dir compare="Text">composite-prefix</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
</test-suite>
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/ConcatenatingTupleReference.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/ConcatenatingTupleReference.java
index 18de540..c635411 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/ConcatenatingTupleReference.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/ConcatenatingTupleReference.java
@@ -19,8 +19,6 @@
package org.apache.hyracks.storage.am.common.tuples;
-import java.util.Arrays;
-
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
/**
@@ -96,14 +94,17 @@
return tuples[tupleIndex].getFieldLength(fieldIndex);
}
+ /**
+ * Right now this class is only used by inverted index, and only contains
2 tuples.
+ * As a result, sequential search would be more efficient than binary
search
+ */
private int getTupleIndex(int fIdx) {
- int tupleIndex = Arrays.binarySearch(fieldCounts, 0, numTuples - 1,
fIdx);
- if (tupleIndex < 0) {
- tupleIndex = -tupleIndex - 1;
- } else {
- ++tupleIndex;
+ for (int i = 0; i < numTuples; i++) {
+ if (fIdx < fieldCounts[i]) {
+ return i;
+ }
}
- return tupleIndex;
+ throw new IllegalArgumentException("Illegal field index " + fIdx);
}
private int getFieldIndex(int tupleIndex, int fIdx) {
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java
index 445a005..fb1fa63 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java
@@ -125,7 +125,6 @@
rangeCursors[i].close();
}
}
- rangeCursors = null;
} finally {
if (lsmHarness != null) {
lsmHarness.endSearch(opCtx);
@@ -249,7 +248,7 @@
public class PriorityQueueElement {
private ITupleReference tuple;
- private int cursorIndex;
+ private final int cursorIndex;
public PriorityQueueElement(int cursorIndex) {
tuple = null;
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java
index 4eb7728..f2fb9af 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java
@@ -95,14 +95,14 @@
opTrackerProvider.getOperationTracker(serviceCtx, this),
ioScheduler, ioOpCallbackFactory,
invertedIndexFields, filterTypeTraits, filterCmpFactories,
filterFields,
filterFieldsForNonBulkLoadOps,
invertedIndexFieldsForNonBulkLoadOps, durable,
- metadataPageManagerFactory);
+ metadataPageManagerFactory, serviceCtx.getTracer());
} else {
return InvertedIndexUtils.createLSMInvertedIndex(ioManager,
virtualBufferCaches, typeTraits, cmpFactories,
tokenTypeTraits, tokenCmpFactories, tokenizerFactory,
bufferCache, file.getAbsolutePath(),
bloomFilterFalsePositiveRate, mergePolicy,
opTrackerProvider.getOperationTracker(serviceCtx, this),
ioScheduler, ioOpCallbackFactory, invertedIndexFields,
filterTypeTraits, filterCmpFactories,
filterFields, filterFieldsForNonBulkLoadOps,
invertedIndexFieldsForNonBulkLoadOps, durable,
- metadataPageManagerFactory);
+ metadataPageManagerFactory, serviceCtx.getTracer());
}
}
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
index 5fda514..e46c24a 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
@@ -100,10 +100,10 @@
IBinaryTokenizerFactory tokenizerFactory, ILSMMergePolicy
mergePolicy, ILSMOperationTracker opTracker,
ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallbackFactory ioOpCallbackFactory,
int[] invertedIndexFields, int[] filterFields, int[]
filterFieldsForNonBulkLoadOps,
- int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable)
throws HyracksDataException {
+ int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable,
ITracer tracer) throws HyracksDataException {
super(ioManager, virtualBufferCaches, diskBufferCache, fileManager,
bloomFilterFalsePositiveRate, mergePolicy,
opTracker, ioScheduler, ioOpCallbackFactory, componentFactory,
componentFactory, filterFrameFactory,
- filterManager, filterFields, durable, filterHelper,
invertedIndexFields, ITracer.NONE);
+ filterManager, filterFields, durable, filterHelper,
invertedIndexFields, tracer);
this.tokenizerFactory = tokenizerFactory;
this.invListTypeTraits = invListTypeTraits;
this.invListCmpFactories = invListCmpFactories;
@@ -349,65 +349,65 @@
LSMInvertedIndexMergeOperation mergeOp =
(LSMInvertedIndexMergeOperation) operation;
RangePredicate mergePred = new RangePredicate(null, null, true, true,
null, null);
IIndexCursor cursor = mergeOp.getCursor();
- try {
- ILSMIndexOperationContext opCtx = ((LSMIndexSearchCursor)
cursor).getOpCtx();
- // Scan diskInvertedIndexes ignoring the memoryInvertedIndex.
- // Create an inverted index instance.
- ILSMDiskComponent component =
createDiskComponent(componentFactory, mergeOp.getTarget(),
- mergeOp.getDeletedKeysBTreeTarget(),
mergeOp.getBloomFilterTarget(), true);
- ILSMDiskComponentBulkLoader componentBulkLoader;
- // In case we must keep the deleted-keys BTrees, then they must be
merged *before* merging the inverted
- // indexes so that lsmHarness.endSearch() is called once when the
inverted indexes have been merged.
- if
(mergeOp.getMergingComponents().get(mergeOp.getMergingComponents().size() - 1)
!= diskComponents
- .get(diskComponents.size() - 1)) {
- // Keep the deleted tuples since the oldest disk component is
not included in the merge operation
- LSMInvertedIndexDeletedKeysBTreeMergeCursor btreeCursor =
- new LSMInvertedIndexDeletedKeysBTreeMergeCursor(opCtx);
- try {
- long numElements = 0L;
- for (int i = 0; i < mergeOp.getMergingComponents().size();
++i) {
- numElements += ((LSMInvertedIndexDiskComponent)
mergeOp.getMergingComponents().get(i))
- .getBloomFilter().getNumElements();
- }
- componentBulkLoader = component.createBulkLoader(1.0f,
false, numElements, false, false, false);
- loadDeleteTuples(opCtx, btreeCursor, mergePred,
componentBulkLoader);
- } finally {
- btreeCursor.destroy();
- }
- } else {
- componentBulkLoader = component.createBulkLoader(1.0f, false,
0L, false, false, false);
- }
- search(opCtx, cursor, mergePred);
+ ILSMIndexOperationContext opCtx = ((LSMIndexSearchCursor)
cursor).getOpCtx();
+ // Scan diskInvertedIndexes ignoring the memoryInvertedIndex.
+ // Create an inverted index instance.
+ ILSMDiskComponent component = createDiskComponent(componentFactory,
mergeOp.getTarget(),
+ mergeOp.getDeletedKeysBTreeTarget(),
mergeOp.getBloomFilterTarget(), true);
+ ILSMDiskComponentBulkLoader componentBulkLoader;
+ // In case we must keep the deleted-keys BTrees, then they must be
merged *before* merging the inverted
+ // indexes so that lsmHarness.endSearch() is called once when the
inverted indexes have been merged.
+ if
(mergeOp.getMergingComponents().get(mergeOp.getMergingComponents().size() - 1)
!= diskComponents
+ .get(diskComponents.size() - 1)) {
+ // Keep the deleted tuples since the oldest disk component is not
included in the merge operation
+ LSMInvertedIndexDeletedKeysBTreeMergeCursor btreeCursor =
+ new LSMInvertedIndexDeletedKeysBTreeMergeCursor(opCtx);
try {
- while (cursor.hasNext()) {
- cursor.next();
- ITupleReference tuple = cursor.getTuple();
- componentBulkLoader.add(tuple);
- }
- } finally {
- cursor.close();
- }
- if (component.getLSMComponentFilter() != null) {
- List<ITupleReference> filterTuples = new ArrayList<>();
+ long numElements = 0L;
for (int i = 0; i < mergeOp.getMergingComponents().size();
++i) {
- ITupleReference min =
mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple();
- ITupleReference max =
mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple();
- if (min != null) {
- filterTuples.add(min);
- }
- if (max != null) {
- filterTuples.add(max);
- }
+ numElements += ((LSMInvertedIndexDiskComponent)
mergeOp.getMergingComponents().get(i))
+ .getBloomFilter().getNumElements();
}
-
getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples,
- NoOpOperationCallback.INSTANCE);
-
getFilterManager().writeFilter(component.getLSMComponentFilter(),
component.getMetadataHolder());
+ componentBulkLoader = component.createBulkLoader(1.0f, false,
numElements, false, false, false);
+ loadDeleteTuples(opCtx, btreeCursor, mergePred,
componentBulkLoader);
+ } finally {
+ btreeCursor.destroy();
}
- componentBulkLoader.end();
- return component;
- } finally {
- cursor.close();
+ } else {
+ componentBulkLoader = component.createBulkLoader(1.0f, false, 0L,
false, false, false);
}
+ search(opCtx, cursor, mergePred);
+ try {
+ while (cursor.hasNext()) {
+ cursor.next();
+ componentBulkLoader.add(cursor.getTuple());
+ }
+ } finally {
+ try {
+ cursor.close();
+ } finally {
+ cursor.destroy();
+ }
+ }
+ if (component.getLSMComponentFilter() != null) {
+ List<ITupleReference> filterTuples = new ArrayList<>();
+ for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
+ ITupleReference min =
mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple();
+ ITupleReference max =
mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple();
+ if (min != null) {
+ filterTuples.add(min);
+ }
+ if (max != null) {
+ filterTuples.add(max);
+ }
+ }
+ getFilterManager().updateFilter(component.getLSMComponentFilter(),
filterTuples,
+ NoOpOperationCallback.INSTANCE);
+ getFilterManager().writeFilter(component.getLSMComponentFilter(),
component.getMetadataHolder());
+ }
+ componentBulkLoader.end();
+
+ return component;
}
private void loadDeleteTuples(ILSMIndexOperationContext opCtx,
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDeletedKeysBTreeMergeCursor.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDeletedKeysBTreeMergeCursor.java
index f1f5241..faa90eb 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDeletedKeysBTreeMergeCursor.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDeletedKeysBTreeMergeCursor.java
@@ -60,7 +60,6 @@
ArrayList<IIndexAccessor> btreeAccessors =
lsmInitialState.getDeletedKeysBTreeAccessors();
for (int i = 0; i < numBTrees; i++) {
rangeCursors[i] = btreeAccessors.get(i).createSearchCursor(false);
- btreeAccessors.get(i).search(rangeCursors[i], btreePredicate);
}
IndexCursorUtils.open(btreeAccessors, rangeCursors, btreePredicate);
try {
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java
index 12dc23f..020b788 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java
@@ -98,10 +98,9 @@
keysOnlyTuple.reset(checkElement.getTuple());
int end = checkElement.getCursorIndex();
for (int i = 0; i < end; i++) {
- if (bloomFilters[i] != null &&
bloomFilters[i].contains(keysOnlyTuple, hashes)) {
+ if (bloomFilters[i] != null &&
!bloomFilters[i].contains(keysOnlyTuple, hashes)) {
continue;
}
- deletedKeysBTreeCursors[i].close();
deletedKeysBTreeAccessors.get(i).search(deletedKeysBTreeCursors[i],
keySearchPred);
try {
if (deletedKeysBTreeCursors[i].hasNext()) {
@@ -114,4 +113,31 @@
return false;
}
+ @Override
+ public void doClose() throws HyracksDataException {
+ try {
+ super.doClose();
+ } finally {
+ if (deletedKeysBTreeCursors != null) {
+ for (int i = 0; i < deletedKeysBTreeCursors.length; i++) {
+ deletedKeysBTreeCursors[i].close();
+ }
+ }
+ }
+ }
+
+ @Override
+ public void doDestroy() throws HyracksDataException {
+ try {
+ super.doDestroy();
+ } finally {
+ if (deletedKeysBTreeCursors != null) {
+ for (int i = 0; i < deletedKeysBTreeCursors.length; i++) {
+ deletedKeysBTreeCursors[i].destroy();
+ }
+ deletedKeysBTreeCursors = null;
+ }
+ }
+ }
+
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java
index d7482a8..604d374 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java
@@ -40,6 +40,7 @@
import
org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
import org.apache.hyracks.storage.am.lsm.invertedindex.util.InvertedIndexUtils;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.util.trace.ITracer;
public class PartitionedLSMInvertedIndex extends LSMInvertedIndex {
@@ -52,12 +53,12 @@
IBinaryTokenizerFactory tokenizerFactory, ILSMMergePolicy
mergePolicy, ILSMOperationTracker opTracker,
ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallbackFactory ioOpCallbackFactory,
int[] invertedIndexFields, int[] filterFields, int[]
filterFieldsForNonBulkLoadOps,
- int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable)
throws HyracksDataException {
+ int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable,
ITracer tracer) throws HyracksDataException {
super(ioManager, virtualBufferCaches, componentFactory, filterHelper,
filterFrameFactory, filterManager,
bloomFilterFalsePositiveRate, diskBufferCache, fileManager,
invListTypeTraits, invListCmpFactories,
tokenTypeTraits, tokenCmpFactories, tokenizerFactory,
mergePolicy, opTracker, ioScheduler,
ioOpCallbackFactory, invertedIndexFields, filterFields,
filterFieldsForNonBulkLoadOps,
- invertedIndexFieldsForNonBulkLoadOps, durable);
+ invertedIndexFieldsForNonBulkLoadOps, durable, tracer);
}
@Override
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListCursor.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListCursor.java
index da3f079..7f3d12f 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListCursor.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListCursor.java
@@ -220,9 +220,11 @@
// Assumption: processing inverted list takes time; so, we don't
want to keep them on the buffer cache.
// Rather, we utilize the assigned working memory (buffers).
tmpBuffer = page.getBuffer();
- tmpBuffer.rewind();
- buffers.get(currentBufferIdx).rewind();
- buffers.get(currentBufferIdx).put(tmpBuffer);
+
+ // Copies the entire content of the page to the current buffer in
the working memory.
+ System.arraycopy(tmpBuffer.array(), 0,
buffers.get(currentBufferIdx).array(), 0,
+ buffers.get(currentBufferIdx).capacity());
+
buffers.get(currentBufferIdx).position(buffers.get(currentBufferIdx).capacity());
currentBufferIdx++;
bufferCache.unpin(page);
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListScanCursor.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListScanCursor.java
index b033b99..2401c67 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListScanCursor.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListScanCursor.java
@@ -53,6 +53,7 @@
protected ICachedPage page;
protected boolean pinned;
+ protected int pinnedPageId = -1;
public FixedSizeElementInvertedListScanCursor(IBufferCache bufferCache,
int fileId, ITypeTraits[] invListFields)
throws HyracksDataException {
@@ -103,14 +104,18 @@
*/
@Override
public void loadPages() throws HyracksDataException {
- if (pinned) {
- unloadPages();
- }
if (currentPageId == endPageId) {
+ // inverted list exhausted, return
return;
}
currentPageId++;
+ if (pinned && pinnedPageId == currentPageId) {
+ // already pinned, return
+ return;
+ }
+ unloadPages();
page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId,
currentPageId), false);
+ pinnedPageId = currentPageId;
pinned = true;
}
@@ -134,7 +139,6 @@
// Deducts 1 since the startPage would be set to bufferCurrentPageId +
1 in loadPages().
this.currentPageId = startPageId - 1;
this.numPages = endPageId - startPageId + 1;
- this.pinned = false;
}
@Override
@@ -158,16 +162,14 @@
@Override
public void doClose() throws HyracksDataException {
- if (pinned) {
- unloadPages();
- }
+ // No op
+ // We allow the inverted list cursor to hold at most one page to avoid
+ // unnecessary pins
}
@Override
public void doDestroy() throws HyracksDataException {
- if (pinned) {
- unloadPages();
- }
+ unloadPages();
}
@Override
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
index 2f4f1d6..14ebe46 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
@@ -204,9 +204,9 @@
try {
if (ctx.getBtreeCursor().hasNext()) {
ctx.getBtreeCursor().next();
- openInvertedListCursor(ctx.getBtreeCursor().getTuple(),
listCursor);
+ openInvertedListCursor(ctx.getBtreeCursor().getTuple(),
listCursor, ctx);
} else {
- LSMInvertedIndexSearchCursorInitialState initState = new
LSMInvertedIndexSearchCursorInitialState();
+ LSMInvertedIndexSearchCursorInitialState initState =
ctx.getCursorInitialState();
initState.setInvertedListInfo(0, 0, 0, 0);
listCursor.open(initState, null);
}
@@ -215,8 +215,8 @@
}
}
- public void openInvertedListCursor(ITupleReference btreeTuple,
InvertedListCursor listCursor)
- throws HyracksDataException {
+ public void openInvertedListCursor(ITupleReference btreeTuple,
InvertedListCursor listCursor,
+ OnDiskInvertedIndexOpContext opCtx) throws HyracksDataException {
int startPageId =
IntegerPointable.getInteger(btreeTuple.getFieldData(invListStartPageIdField),
btreeTuple.getFieldStart(invListStartPageIdField));
int endPageId =
IntegerPointable.getInteger(btreeTuple.getFieldData(invListEndPageIdField),
@@ -225,7 +225,7 @@
btreeTuple.getFieldStart(invListStartOffField));
int numElements =
IntegerPointable.getInteger(btreeTuple.getFieldData(invListNumElementsField),
btreeTuple.getFieldStart(invListNumElementsField));
- LSMInvertedIndexSearchCursorInitialState initState = new
LSMInvertedIndexSearchCursorInitialState();
+ LSMInvertedIndexSearchCursorInitialState initState =
opCtx.getCursorInitialState();
initState.setInvertedListInfo(startPageId, endPageId, startOff,
numElements);
listCursor.open(initState, null);
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexOpContext.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexOpContext.java
index 267cc79..81bc46b9 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexOpContext.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexOpContext.java
@@ -25,6 +25,7 @@
import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
+import
org.apache.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndexSearchCursorInitialState;
import org.apache.hyracks.storage.common.IIndexAccessor;
import org.apache.hyracks.storage.common.IIndexCursor;
import org.apache.hyracks.storage.common.MultiComparator;
@@ -32,12 +33,13 @@
public class OnDiskInvertedIndexOpContext implements IIndexOperationContext {
private final RangePredicate btreePred = new RangePredicate(null, null,
true, true, null, null);
- private IIndexAccessor btreeAccessor;
- private IIndexCursor btreeCursor;
- private MultiComparator searchCmp;
+ private final IIndexAccessor btreeAccessor;
+ private final IIndexCursor btreeCursor;
+ private final MultiComparator searchCmp;
// For prefix search on partitioned indexes.
private MultiComparator prefixSearchCmp;
private boolean destroyed = false;
+ private LSMInvertedIndexSearchCursorInitialState cursorInitialState;
public OnDiskInvertedIndexOpContext(BTree btree) throws
HyracksDataException {
// TODO: Ignore opcallbacks for now.
@@ -96,4 +98,11 @@
btreeCursor.destroy();
}
}
+
+ public LSMInvertedIndexSearchCursorInitialState getCursorInitialState() {
+ if (cursorInitialState == null) {
+ cursorInitialState = new
LSMInvertedIndexSearchCursorInitialState();
+ }
+ return cursorInitialState;
+ }
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java
index d9e7d34..9d99c9e 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java
@@ -26,7 +26,6 @@
import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters;
import org.apache.hyracks.storage.am.common.tuples.ConcatenatingTupleReference;
import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
-import
org.apache.hyracks.storage.am.lsm.invertedindex.api.IInPlaceInvertedIndex;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.InvertedListCursor;
import org.apache.hyracks.storage.common.EnforcedIndexCursor;
import org.apache.hyracks.storage.common.ICursorInitialState;
@@ -41,7 +40,7 @@
private final BTree btree;
private final IIndexAccessor btreeAccessor;
- private final IInPlaceInvertedIndex invIndex;
+ private final OnDiskInvertedIndex invIndex;
private final IIndexOperationContext opCtx;
private final InvertedListCursor invListRangeSearchCursor;
private boolean isInvListCursorOpen;
@@ -50,11 +49,11 @@
private RangePredicate btreePred;
private final PermutingTupleReference tokenTuple;
- private ConcatenatingTupleReference concatTuple;
+ private final ConcatenatingTupleReference concatTuple;
- public OnDiskInvertedIndexRangeSearchCursor(IInPlaceInvertedIndex
invIndex, IIndexOperationContext opCtx)
+ public OnDiskInvertedIndexRangeSearchCursor(OnDiskInvertedIndex invIndex,
IIndexOperationContext opCtx)
throws HyracksDataException {
- this.btree = ((OnDiskInvertedIndex) invIndex).getBTree();
+ this.btree = invIndex.getBTree();
this.btreeAccessor =
btree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
this.invIndex = invIndex;
this.opCtx = opCtx;
@@ -87,11 +86,7 @@
return true;
}
// The current inverted-list-range-search cursor is exhausted.
- try {
- invListRangeSearchCursor.unloadPages();
- } finally {
- invListRangeSearchCursor.close();
- }
+ invListRangeSearchCursor.close();
isInvListCursorOpen = false;
openInvListRangeSearchCursor();
return isInvListCursorOpen;
@@ -109,14 +104,8 @@
@Override
public void doDestroy() throws HyracksDataException {
try {
- if (isInvListCursorOpen) {
- try {
- invListRangeSearchCursor.unloadPages();
- } finally {
- isInvListCursorOpen = false;
- invListRangeSearchCursor.destroy();
- }
- }
+ invListRangeSearchCursor.destroy();
+ isInvListCursorOpen = false;
} finally {
btreeCursor.destroy();
}
@@ -125,14 +114,8 @@
@Override
public void doClose() throws HyracksDataException {
try {
- if (isInvListCursorOpen) {
- try {
- invListRangeSearchCursor.unloadPages();
- } finally {
- invListRangeSearchCursor.close();
- }
- isInvListCursorOpen = false;
- }
+ invListRangeSearchCursor.close();
+ isInvListCursorOpen = false;
} finally {
btreeCursor.close();
}
@@ -148,7 +131,8 @@
if (btreeCursor.hasNext()) {
btreeCursor.next();
tokenTuple.reset(btreeCursor.getTuple());
- invIndex.openInvertedListCursor(invListRangeSearchCursor,
tokenTuple, opCtx);
+ invIndex.openInvertedListCursor(btreeCursor.getTuple(),
invListRangeSearchCursor,
+ (OnDiskInvertedIndexOpContext) opCtx);
invListRangeSearchCursor.prepareLoadPages();
invListRangeSearchCursor.loadPages();
concatTuple.reset();
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java
index eff4f5a..8c6b386 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java
@@ -117,7 +117,7 @@
short numTokens =
ShortPointable.getShort(btreeTuple.getFieldData(PARTITIONING_NUM_TOKENS_FIELD),
btreeTuple.getFieldStart(PARTITIONING_NUM_TOKENS_FIELD));
InvertedListCursor invListCursor =
partSearcher.getCachedInvertedListCursor();
- openInvertedListCursor(btreeTuple, invListCursor);
+ openInvertedListCursor(btreeTuple, invListCursor, ctx);
invListPartitions.addInvertedListCursor(invListCursor,
numTokens);
tokenExists = true;
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
index ba54ed4..1e812b4 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
@@ -62,6 +62,7 @@
import
org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.PartitionedOnDiskInvertedIndexFactory;
import
org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.util.trace.ITracer;
public class InvertedIndexUtils {
@@ -129,7 +130,7 @@
ILSMIOOperationCallbackFactory ioOpCallbackFactory, int[]
invertedIndexFields,
ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[]
filterCmpFactories, int[] filterFields,
int[] filterFieldsForNonBulkLoadOps, int[]
invertedIndexFieldsForNonBulkLoadOps, boolean durable,
- IMetadataPageManagerFactory pageManagerFactory) throws
HyracksDataException {
+ IMetadataPageManagerFactory pageManagerFactory, ITracer tracer)
throws HyracksDataException {
BTreeFactory deletedKeysBTreeFactory =
createDeletedKeysBTreeFactory(ioManager, invListTypeTraits,
invListCmpFactories, diskBufferCache, pageManagerFactory);
@@ -166,7 +167,7 @@
filterManager, bloomFilterFalsePositiveRate, diskBufferCache,
fileManager, invListTypeTraits,
invListCmpFactories, tokenTypeTraits, tokenCmpFactories,
tokenizerFactory, mergePolicy, opTracker,
ioScheduler, ioOpCallbackFactory, invertedIndexFields,
filterFields, filterFieldsForNonBulkLoadOps,
- invertedIndexFieldsForNonBulkLoadOps, durable);
+ invertedIndexFieldsForNonBulkLoadOps, durable, tracer);
}
public static PartitionedLSMInvertedIndex
createPartitionedLSMInvertedIndex(IIOManager ioManager,
@@ -178,7 +179,7 @@
ILSMIOOperationCallbackFactory ioOpCallbackFactory, int[]
invertedIndexFields,
ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[]
filterCmpFactories, int[] filterFields,
int[] filterFieldsForNonBulkLoadOps, int[]
invertedIndexFieldsForNonBulkLoadOps, boolean durable,
- IPageManagerFactory pageManagerFactory) throws
HyracksDataException {
+ IPageManagerFactory pageManagerFactory, ITracer tracer) throws
HyracksDataException {
BTreeFactory deletedKeysBTreeFactory =
createDeletedKeysBTreeFactory(ioManager, invListTypeTraits,
invListCmpFactories, diskBufferCache, pageManagerFactory);
@@ -215,6 +216,6 @@
filterFrameFactory, filterManager,
bloomFilterFalsePositiveRate, diskBufferCache, fileManager,
invListTypeTraits, invListCmpFactories, tokenTypeTraits,
tokenCmpFactories, tokenizerFactory,
mergePolicy, opTracker, ioScheduler, ioOpCallbackFactory,
invertedIndexFields, filterFields,
- filterFieldsForNonBulkLoadOps,
invertedIndexFieldsForNonBulkLoadOps, durable);
+ filterFieldsForNonBulkLoadOps,
invertedIndexFieldsForNonBulkLoadOps, durable, tracer);
}
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
index ae8713c..2435cc4 100644
---
a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
+++
b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
@@ -55,6 +55,9 @@
import
org.apache.hyracks.storage.am.lsm.invertedindex.util.LSMInvertedIndexTestUtils.HyracksTaskTestContext;
import org.apache.hyracks.storage.common.IIndex;
import org.apache.hyracks.storage.common.IIndexAccessParameters;
+import org.apache.hyracks.util.trace.ITraceCategoryRegistry;
+import org.apache.hyracks.util.trace.TraceCategoryRegistry;
+import org.apache.hyracks.util.trace.Tracer;
@SuppressWarnings("rawtypes")
public class LSMInvertedIndexTestContext extends OrderedIndexTestContext {
@@ -184,7 +187,9 @@
harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(),
harness.getIOOperationCallbackFactory(),
invertedIndexFields, filterTypeTraits,
filterCmpFactories, filterFields,
filterFieldsForNonBulkLoadOps,
- invertedIndexFieldsForNonBulkLoadOps, true,
harness.getMetadataPageManagerFactory());
+ invertedIndexFieldsForNonBulkLoadOps, true,
harness.getMetadataPageManagerFactory(),
+ new
Tracer(LSMInvertedIndexTestContext.class.getSimpleName(),
+ ITraceCategoryRegistry.CATEGORIES_ALL, new
TraceCategoryRegistry()));
break;
}
case PARTITIONED_LSM: {
@@ -195,7 +200,9 @@
harness.getOperationTracker(),
harness.getIOScheduler(),
harness.getIOOperationCallbackFactory(),
invertedIndexFields, filterTypeTraits,
filterCmpFactories, filterFields,
filterFieldsForNonBulkLoadOps,
- invertedIndexFieldsForNonBulkLoadOps, true,
harness.getMetadataPageManagerFactory());
+ invertedIndexFieldsForNonBulkLoadOps, true,
harness.getMetadataPageManagerFactory(),
+ new
Tracer(LSMInvertedIndexTestContext.class.getSimpleName(),
+ ITraceCategoryRegistry.CATEGORIES_ALL, new
TraceCategoryRegistry()));
break;
}
default: {
--
To view, visit https://asterix-gerrit.ics.uci.edu/2531
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I6dbbc1b7b78762737a21773efc53035fc62d8f24
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: release-0.9.4-pre-rc
Gerrit-Owner: Luo Chen <[email protected]>
