This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 62578ad6659762f5e7fda13421abe9c38cae3605 Author: Peeyush Gupta <[email protected]> AuthorDate: Thu Nov 30 15:17:31 2023 -0800 [ASTERIXDB-3323][COMP] Array index bulkload not indexing all values - user model changes: no - storage format changes: no - interface changes: no Change-Id: Ifc936ae867e90fff023f89ef2d21006e2e4f895d Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17989 Reviewed-by: Ali Alsuliman <[email protected]> Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> --- .../index-bad-fields-bulkload.001.update.sqlpp | 28 ---------------------- .../index-bad-fields-bulkload.005.ddl.sqlpp | 22 ----------------- .../index-bad-fields-bulkload.999.ddl.sqlpp | 20 ---------------- .../index-bad-fields.008.ddl.sqlpp} | 8 ++++--- .../index-bad-fields.009.query.sqlpp} | 2 +- .../index-bad-fields.010.query.sqlpp} | 5 +++- .../index-bad-fields.011.query.sqlpp} | 5 +++- .../index-bad-fields.012.query.sqlpp} | 5 +++- .../index-bad-fields.013.query.sqlpp} | 2 +- .../index-bad-fields/index-bad-fields.009.adm | 10 ++++++++ .../index-bad-fields/index-bad-fields.010.adm | 2 ++ .../index-bad-fields/index-bad-fields.011.adm | 1 + .../index-bad-fields/index-bad-fields.012.adm | 2 ++ .../index-bad-fields/index-bad-fields.013.adm | 1 + .../test/resources/runtimets/testsuite_sqlpp.xml | 13 ---------- .../asterix/metadata/utils/ArrayIndexUtil.java | 4 +++- .../SecondaryArrayIndexBTreeOperationsHelper.java | 18 +++++++------- .../utils/SecondaryIndexOperationsHelper.java | 21 +++++++++++++++- 18 files changed, 68 insertions(+), 101 deletions(-) diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.001.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.001.update.sqlpp deleted file mode 100644 index 1822abea37..0000000000 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.001.update.sqlpp +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -USE test; -UPSERT INTO ds1 {"id": 1, "a": ["hello", 1]}; -UPSERT INTO ds1 {"id": 2, "b": ["hello", "hello"]}; -UPSERT INTO ds1 {"id": 3, "b": [11, 12]}; -UPSERT INTO ds1 {"id": 4, "c": [10, 10]}; - -UPSERT INTO ds1 {"id": 5, "d": [{"x":1}, {"x":"A"}]}; -UPSERT INTO ds1 {"id": 6, "d": [{"x":1}, {"x":"A"}]}; - diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.005.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.005.ddl.sqlpp deleted file mode 100644 index ee3de29f18..0000000000 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.005.ddl.sqlpp +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -USE test; - -CREATE INDEX i4 ON ds1(UNNEST c:int) EXCLUDE UNKNOWN KEY; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.999.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.999.ddl.sqlpp deleted file mode 100644 index 86a1b59399..0000000000 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.999.ddl.sqlpp +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -DROP DATAVERSE test; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.008.ddl.sqlpp similarity index 68% rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.000.ddl.sqlpp rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.008.ddl.sqlpp index efe6d6b957..7850fde40c 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.000.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.008.ddl.sqlpp @@ -21,8 +21,10 @@ * Expected Res : Said tuples are not indexed */ -CREATE DATAVERSE test; USE test; -CREATE TYPE dt1 as {id:int}; -CREATE DATASET ds1(dt1) primary key id; +CREATE INDEX i6 ON ds1(UNNEST a : string) EXCLUDE UNKNOWN KEY; +CREATE INDEX i7 ON ds1(UNNEST b SELECT x : int) EXCLUDE UNKNOWN KEY; +CREATE INDEX i8 ON ds1(UNNEST b.x SELECT p: int) EXCLUDE UNKNOWN KEY; +CREATE INDEX i9 ON ds1(UNNEST b.c UNNEST d.e UNNEST t SELECT x.y : string, q.w: string, u: int) EXCLUDE UNKNOWN KEY; +CREATE INDEX i10 ON ds1(p: int, (UNNEST b.c UNNEST d.e UNNEST t SELECT x.y : string, q.w: string), z.m: double) EXCLUDE UNKNOWN KEY; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.006.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.009.query.sqlpp similarity index 93% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.006.query.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.009.query.sqlpp index dcf23cf721..a33bb2368e 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.006.query.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.009.query.sqlpp @@ -20,6 +20,6 @@ USE test; SET `import-private-functions` `true`; -FROM DUMP_INDEX("test", "ds1", "i4") AS v +FROM DUMP_INDEX("test", "ds1", "i6") AS v SELECT VALUE v ORDER BY v.values; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.004.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.010.query.sqlpp similarity index 87% rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.004.ddl.sqlpp rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.010.query.sqlpp index 70119b54a6..4134554ab3 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.004.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.010.query.sqlpp @@ -19,4 +19,7 @@ USE test; -CREATE INDEX i2 ON ds1(UNNEST d select x:string) EXCLUDE UNKNOWN KEY; +SET `import-private-functions` `true`; +FROM DUMP_INDEX("test", "ds1", "i7") AS v +SELECT VALUE v +ORDER BY v.values; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.002.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.011.query.sqlpp similarity index 87% rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.002.ddl.sqlpp rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.011.query.sqlpp index fbe85d356b..75b56d1c7a 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.002.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.011.query.sqlpp @@ -19,4 +19,7 @@ USE test; -CREATE INDEX i2 ON ds1(UNNEST b:int) EXCLUDE UNKNOWN KEY; +SET `import-private-functions` `true`; +FROM DUMP_INDEX("test", "ds1", "i8") AS v +SELECT VALUE v +ORDER BY v.values; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.003.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.012.query.sqlpp similarity index 87% rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.003.ddl.sqlpp rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.012.query.sqlpp index fbe85d356b..5364ebbc47 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.003.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.012.query.sqlpp @@ -19,4 +19,7 @@ USE test; -CREATE INDEX i2 ON ds1(UNNEST b:int) EXCLUDE UNKNOWN KEY; +SET `import-private-functions` `true`; +FROM DUMP_INDEX("test", "ds1", "i9") AS v +SELECT VALUE v +ORDER BY v.values; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.006.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.013.query.sqlpp similarity index 93% rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.006.query.sqlpp rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.013.query.sqlpp index dcf23cf721..44f59c8310 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.006.query.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.013.query.sqlpp @@ -20,6 +20,6 @@ USE test; SET `import-private-functions` `true`; -FROM DUMP_INDEX("test", "ds1", "i4") AS v +FROM DUMP_INDEX("test", "ds1", "i10") AS v SELECT VALUE v ORDER BY v.values; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.009.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.009.adm new file mode 100644 index 0000000000..a4643809ba --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.009.adm @@ -0,0 +1,10 @@ +{ "values": [ "hello", 1 ] } +{ "values": [ "hello", 5 ] } +{ "values": [ "hello", 6 ] } +{ "values": [ "hello", 8 ] } +{ "values": [ "hello", 10 ] } +{ "values": [ "hello", 11 ] } +{ "values": [ "hello", 12 ] } +{ "values": [ "hello", 13 ] } +{ "values": [ "hello", 14 ] } +{ "values": [ "hello", 15 ] } diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.010.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.010.adm new file mode 100644 index 0000000000..3fdbd81ebe --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.010.adm @@ -0,0 +1,2 @@ +{ "values": [ 1, 5 ] } +{ "values": [ 10, 7 ] } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.011.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.011.adm new file mode 100644 index 0000000000..6045d06eb7 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.011.adm @@ -0,0 +1 @@ +{ "values": [ 1, 10 ] } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.012.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.012.adm new file mode 100644 index 0000000000..4c44dbabd7 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.012.adm @@ -0,0 +1,2 @@ +{ "values": [ "aab", "10", 90, 11 ] } +{ "values": [ "aab", "10", 93, 14 ] } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.013.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.013.adm new file mode 100644 index 0000000000..a38bbc602f --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.013.adm @@ -0,0 +1 @@ +{ "values": [ 100, "aab", "10", 100.1, 14 ] } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml index 9a3abae1d0..72578fa2ab 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml @@ -8356,8 +8356,6 @@ <test-case FilePath="array-index/bulk-loading/on-index-creation/open"> <compilation-unit name="composite-atomic"> <output-dir compare="Text">composite-atomic</output-dir> - <expected-error>ASX0001: Field type string cannot be promoted to type bigint</expected-error> - <source-location>false</source-location> </compilation-unit> </test-case> </test-group> @@ -8662,17 +8660,6 @@ </compilation-unit> </test-case> </test-group> - <test-group name="array-index/index-bad-fields-bulkload"> - <test-case FilePath="array-index"> - <compilation-unit name="index-bad-fields-bulkload"> - <output-dir compare="Text">index-bad-fields-bulkload</output-dir> - <expected-error>ASX0018: Cannot convert the string type to the bigint type</expected-error> - <expected-error>ASX0018: Cannot convert the string type to the bigint type</expected-error> - <expected-error>ASX0001: Field type bigint cannot be promoted to type string</expected-error> - <source-location>false</source-location> - </compilation-unit> - </test-case> - </test-group> </test-group> <test-group name="nestrecords"> <test-case FilePath="nestrecords"> diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java index 7332dd0aa0..498630a13f 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java @@ -35,6 +35,7 @@ import org.apache.asterix.om.types.AUnorderedListType; import org.apache.asterix.om.types.AbstractCollectionType; import org.apache.asterix.om.types.IAType; import org.apache.asterix.om.utils.NonTaggedFormatUtil; +import org.apache.asterix.om.utils.RecordUtil; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.common.utils.Pair; @@ -254,7 +255,8 @@ public class ArrayIndexUtil { IAType workingType = baseRecordType; for (int i = 0; i < fieldNamesPerArray.size(); i++) { - ARecordType startingStepRecordType = (isTrackingType) ? (ARecordType) workingType : null; + ARecordType startingStepRecordType = + (isTrackingType) ? (ARecordType) workingType : RecordUtil.FULLY_OPEN_RECORD_TYPE; if (isTrackingType) { if (!workingType.getTypeTag().equals(ATypeTag.OBJECT)) { throw new AsterixException(ErrorCode.COMPILATION_ERROR, "Mismatched record type to depth-" diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryArrayIndexBTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryArrayIndexBTreeOperationsHelper.java index af0fd82ed2..de0ca5af74 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryArrayIndexBTreeOperationsHelper.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryArrayIndexBTreeOperationsHelper.java @@ -158,7 +158,7 @@ public class SecondaryArrayIndexBTreeOperationsHelper extends SecondaryTreeIndex int flattenedListPos = 0; for (Index.ArrayIndexElement e : arrayIndexDetails.getElementList()) { for (int i = 0; i < e.getProjectList().size(); i++) { - addSKEvalFactories(isOverridingKeyFieldTypes ? enforcedItemType : itemType, flattenedListPos, false, e); + addSKEvalFactories(itemType, flattenedListPos, false, e); Pair<IAType, Boolean> keyTypePair = ArrayIndexUtil.getNonNullableOpenFieldType(e.getTypeList().get(i), e.getUnnestList(), e.getProjectList().get(i), itemType); IAType keyType = keyTypePair.first; @@ -268,12 +268,6 @@ public class SecondaryArrayIndexBTreeOperationsHelper extends SecondaryTreeIndex spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0); sourceOp = targetOp; - if (arrayIndexDetails.isOverridingKeyFieldTypes() && !enforcedItemType.equals(itemType)) { - // If we have an enforced type, insert a "cast" after the primary index scan. - targetOp = createCastOp(spec, dataset.getDatasetType(), true); - spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0); - sourceOp = targetOp; - } // We do not index meta fields. Project away meta fields if they exist. if (dataset.hasMetaPart()) { @@ -306,7 +300,15 @@ public class SecondaryArrayIndexBTreeOperationsHelper extends SecondaryTreeIndex if (anySecondaryKeyIsNullable || arrayIndexDetails.isOverridingKeyFieldTypes()) { // If any of the secondary fields are nullable, then we need to filter out the nulls. - targetOp = createFilterAnyUnknownSelectOp(spec, numTotalSecondaryKeys, secondaryRecDesc); + List<IAType> secondaryKeyTypes = new ArrayList<>(); + if (arrayIndexDetails.isOverridingKeyFieldTypes() && !enforcedItemType.equals(itemType)) { + for (Index.ArrayIndexElement arrayIndexElement : arrayIndexDetails.getElementList()) { + List<IAType> typeList = arrayIndexElement.getTypeList(); + secondaryKeyTypes.addAll(typeList); + } + } + targetOp = createCastFilterAnyUnknownSelectOp(spec, numTotalSecondaryKeys, secondaryRecDesc, + secondaryKeyTypes); spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0); sourceOp = targetOp; } diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java index 694b153a8b..ab64b18bfe 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java @@ -478,9 +478,22 @@ public abstract class SecondaryIndexOperationsHelper implements ISecondaryIndexO return createFilterSelectOp(spec, numSecondaryKeyFields, secondaryRecDesc, AndDescriptor::new); } + public AlgebricksMetaOperatorDescriptor createCastFilterAnyUnknownSelectOp(JobSpecification spec, + int numSecondaryKeyFields, RecordDescriptor secondaryRecDesc, List<IAType> castFieldTypes) + throws AlgebricksException { + return createFilterSelectOp(spec, numSecondaryKeyFields, secondaryRecDesc, AndDescriptor::new, castFieldTypes); + } + private AlgebricksMetaOperatorDescriptor createFilterSelectOp(JobSpecification spec, int numSecondaryKeyFields, RecordDescriptor secondaryRecDesc, Supplier<AbstractFunctionDescriptor> predicatesCombinerFuncSupplier) throws AlgebricksException { + return createFilterSelectOp(spec, numSecondaryKeyFields, secondaryRecDesc, predicatesCombinerFuncSupplier, + Collections.emptyList()); + } + + private AlgebricksMetaOperatorDescriptor createFilterSelectOp(JobSpecification spec, int numSecondaryKeyFields, + RecordDescriptor secondaryRecDesc, Supplier<AbstractFunctionDescriptor> predicatesCombinerFuncSupplier, + List<IAType> castFieldTypes) throws AlgebricksException { IScalarEvaluatorFactory[] predicateArgsEvalFactories = new IScalarEvaluatorFactory[numSecondaryKeyFields]; NotDescriptor notDesc = new NotDescriptor(); notDesc.setSourceLocation(sourceLoc); @@ -489,8 +502,14 @@ public abstract class SecondaryIndexOperationsHelper implements ISecondaryIndexO for (int i = 0; i < numSecondaryKeyFields; i++) { // Access column i, and apply 'is not null'. ColumnAccessEvalFactory columnAccessEvalFactory = new ColumnAccessEvalFactory(i); + IScalarEvaluatorFactory evalFactory = columnAccessEvalFactory; + if (castFieldTypes != null && !castFieldTypes.isEmpty()) { + IScalarEvaluatorFactory[] castArg = new IScalarEvaluatorFactory[] { columnAccessEvalFactory }; + evalFactory = createCastFunction(castFieldTypes.get(i), BuiltinType.ANY, index.isEnforced(), sourceLoc) + .createEvaluatorFactory(castArg); + } IScalarEvaluatorFactory isUnknownEvalFactory = - isUnknownDesc.createEvaluatorFactory(new IScalarEvaluatorFactory[] { columnAccessEvalFactory }); + isUnknownDesc.createEvaluatorFactory(new IScalarEvaluatorFactory[] { evalFactory }); IScalarEvaluatorFactory notEvalFactory = notDesc.createEvaluatorFactory(new IScalarEvaluatorFactory[] { isUnknownEvalFactory }); predicateArgsEvalFactories[i] = notEvalFactory;
