This is an automated email from the ASF dual-hosted git repository. imaxon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 46d36883bf8be3d5866f39d21dae74fdcfbe7203 Author: Ritik Raj <[email protected]> AuthorDate: Sat Feb 28 15:33:34 2026 +0530 [ASTERIXDB-3708][COMP] Fix empty results with columnar secondary index - user model changes: no - storage format changes: no - interface changes: no Details: When querying a columnar dataset with a secondary index, LIMIT clause, and field projection (e.g., SELECT VALUE h FROM d1 WHERE email_domain > 3 LIMIT 2), the query returns empty results with certain execution sequences. Root cause: PushLimitIntoPrimarySearchRule moves the SELECT condition into the UNNEST_MAP operator before field access pushdown. PushdownOperatorVisitor was calling visitInputs BEFORE registering the dataset for UnnestMapOperator and LeftOuterUnnestMapOperator. This timing issue prevented filter fields from being captured in the def-use chain, so the columnar projection schema only included output fields but not filter fields. Fix: Changed PushdownOperatorVisitor to call registerDatasetIfApplicable BEFORE visitInputs for UnnestMapOperator and LeftOuterUnnestMapOperator, aligning with the existing pattern for DataSourceScanOperator. This ensures filter field accesses are properly linked to the scan definition, causing the projection schema to include both output and filter fields. Added regression test in column/secondary-index/projection-filter/ to validate the fix. Ext-ref: MB-70725 Change-Id: I26e5357141ff77ee210f89b4e93e1e4c33eb6845 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20948 Integration-Tests: Jenkins <[email protected]> Reviewed-by: Ali Alsuliman <[email protected]> Reviewed-by: Ritik Raj <[email protected]> Tested-by: Jenkins <[email protected]> --- .../pushdown/visitor/PushdownOperatorVisitor.java | 4 +-- .../projection-filter.001.ddl.sqlpp | 30 ++++++++++++++++++++++ .../projection-filter.002.update.sqlpp | 22 ++++++++++++++++ .../projection-filter.003.get.http | 19 ++++++++++++++ .../projection-filter.005.ddl.sqlpp | 22 ++++++++++++++++ .../projection-filter.006.query.sqlpp | 22 ++++++++++++++++ .../projection-filter.007.query.sqlpp | 22 ++++++++++++++++ .../projection-filter.003.regexadm | 1 + .../projection-filter/projection-filter.006.adm | 1 + .../projection-filter/projection-filter.007.adm | 1 + .../runtimets/testsuite_single_partition_sqlpp.xml | 5 ++++ .../asterix/metadata/utils/PushdownUtil.java | 2 ++ 12 files changed, 149 insertions(+), 2 deletions(-) diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/visitor/PushdownOperatorVisitor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/visitor/PushdownOperatorVisitor.java index c425d72748..7878ebf816 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/visitor/PushdownOperatorVisitor.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/visitor/PushdownOperatorVisitor.java @@ -175,9 +175,9 @@ public class PushdownOperatorVisitor implements ILogicalOperatorVisitor<Void, Vo */ @Override public Void visitUnnestMapOperator(UnnestMapOperator op, Void arg) throws AlgebricksException { - visitInputs(op); DatasetDataSource datasetDataSource = getDatasetDataSourceIfApplicable(getDataSourceFromUnnestMapOperator(op)); registerDatasetIfApplicable(datasetDataSource, op); + visitInputs(op); return null; } @@ -187,9 +187,9 @@ public class PushdownOperatorVisitor implements ILogicalOperatorVisitor<Void, Vo */ @Override public Void visitLeftOuterUnnestMapOperator(LeftOuterUnnestMapOperator op, Void arg) throws AlgebricksException { - visitInputs(op); DatasetDataSource datasetDataSource = getDatasetDataSourceIfApplicable(getDataSourceFromUnnestMapOperator(op)); registerDatasetIfApplicable(datasetDataSource, op); + visitInputs(op); return null; } diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/secondary-index/projection-filter/projection-filter.001.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/secondary-index/projection-filter/projection-filter.001.ddl.sqlpp new file mode 100644 index 0000000000..04c91b6c71 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/secondary-index/projection-filter/projection-filter.001.ddl.sqlpp @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +DROP DATAVERSE projection_filter IF EXISTS; +CREATE DATAVERSE projection_filter; +USE projection_filter; + +CREATE TYPE intType AS { + id: int +}; + +CREATE DATASET d1(intType) PRIMARY KEY id WITH { + "storage-format": {"format": "column"} +}; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/secondary-index/projection-filter/projection-filter.002.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/secondary-index/projection-filter/projection-filter.002.update.sqlpp new file mode 100644 index 0000000000..ad04eae082 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/secondary-index/projection-filter/projection-filter.002.update.sqlpp @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE projection_filter; + +INSERT INTO d1 {"id":2,"h":44,"cac_email_domain":66}; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/secondary-index/projection-filter/projection-filter.003.get.http b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/secondary-index/projection-filter/projection-filter.003.get.http new file mode 100644 index 0000000000..eef4e416b8 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/secondary-index/projection-filter/projection-filter.003.get.http @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/connector?dataverseName=projection_filter&datasetName=d1 diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/secondary-index/projection-filter/projection-filter.005.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/secondary-index/projection-filter/projection-filter.005.ddl.sqlpp new file mode 100644 index 0000000000..00df2b7c20 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/secondary-index/projection-filter/projection-filter.005.ddl.sqlpp @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF any + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE projection_filter; + +CREATE INDEX idx_cac_email_domain ON d1(cac_email_domain); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/secondary-index/projection-filter/projection-filter.006.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/secondary-index/projection-filter/projection-filter.006.query.sqlpp new file mode 100644 index 0000000000..b30c14f5d5 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/secondary-index/projection-filter/projection-filter.006.query.sqlpp @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE projection_filter; + +SELECT VALUE h FROM d1 WHERE cac_email_domain > 3; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/secondary-index/projection-filter/projection-filter.007.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/secondary-index/projection-filter/projection-filter.007.query.sqlpp new file mode 100644 index 0000000000..4d65bc3d32 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/secondary-index/projection-filter/projection-filter.007.query.sqlpp @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE projection_filter; + +SELECT VALUE h FROM d1 WHERE cac_email_domain > 3 LIMIT 2; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/column/secondary-index/projection-filter/projection-filter.003.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/secondary-index/projection-filter/projection-filter.003.regexadm new file mode 100644 index 0000000000..78194e6ef6 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/secondary-index/projection-filter/projection-filter.003.regexadm @@ -0,0 +1 @@ +\Q{"keys":"id","type":{"type":"org.apache.asterix.om.types.ARecordType","name":"intType","open":true,"fields":[{"id":{"type":"AInt64"}}]},"splits":[\E.*\Q]}\E \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/column/secondary-index/projection-filter/projection-filter.006.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/secondary-index/projection-filter/projection-filter.006.adm new file mode 100644 index 0000000000..d2e1cefe89 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/secondary-index/projection-filter/projection-filter.006.adm @@ -0,0 +1 @@ +44 \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/column/secondary-index/projection-filter/projection-filter.007.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/secondary-index/projection-filter/projection-filter.007.adm new file mode 100644 index 0000000000..d2e1cefe89 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/secondary-index/projection-filter/projection-filter.007.adm @@ -0,0 +1 @@ +44 \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_single_partition_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_single_partition_sqlpp.xml index b248475c14..d0a8452626 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_single_partition_sqlpp.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_single_partition_sqlpp.xml @@ -404,6 +404,11 @@ <output-dir compare="Text">big-object</output-dir> </compilation-unit> </test-case> + <test-case FilePath="column"> + <compilation-unit name="secondary-index/projection-filter"> + <output-dir compare="Text">secondary-index/projection-filter</output-dir> + </compilation-unit> + </test-case> <test-case FilePath="column"> <compilation-unit name="secondary-index/array-index/use-case-1"> <output-dir compare="Text">secondary-index/array-index/use-case-1</output-dir> diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/PushdownUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/PushdownUtil.java index 316b559b93..3b8f6abe9a 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/PushdownUtil.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/PushdownUtil.java @@ -70,6 +70,8 @@ public class PushdownUtil { public static IVariableTypeEnvironment getTypeEnv(ILogicalOperator useOperator, IOptimizationContext context) throws AlgebricksException { if (useOperator.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN + || useOperator.getOperatorTag() == LogicalOperatorTag.UNNEST_MAP + || useOperator.getOperatorTag() == LogicalOperatorTag.LEFT_OUTER_UNNEST_MAP || useOperator.getOperatorTag() == LogicalOperatorTag.INNERJOIN) { // Special case: for pushed select condition return useOperator.computeOutputTypeEnvironment(context);
