>From Wail Alkowaileet <[email protected]>: Wail Alkowaileet has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17849 )
Change subject: [ASTERIXDB-3276][RT] Fix infinite loop in columnar filter ...................................................................... [ASTERIXDB-3276][RT] Fix infinite loop in columnar filter - user model changes: no - storage format changes: no - interface changes: no Details: Columnar filter can stuck in infinite loop when a non-existing value is part of the filter expression. Change-Id: Ib32c9a1fe2814da4091c96e504d694b67d0d0d56 --- M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/AbstractColumnTupleReference.java M asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/value/NoOpValueReader.java M asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnTupleReference.java A asterixdb/asterix-app/src/test/resources/runtimets/results/column/filter/003/003.006.adm M asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/evaluator/ColumnIterableFilterEvaluatorFactory.java M asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/visitor/PathExtractorVisitor.java A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/filter/003/003.007.query.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/filter/003/003.006.query.sqlpp M asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnWithMetaTupleReference.java A asterixdb/asterix-app/src/test/resources/runtimets/results/column/filter/003/003.007.adm M asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java 11 files changed, 95 insertions(+), 11 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/49/17849/1 diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/filter/003/003.006.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/filter/003/003.006.query.sqlpp new file mode 100644 index 0000000..8d6fa80 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/filter/003/003.006.query.sqlpp @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; +SET `compiler.column.filter` "true"; + +SELECT VALUE d +FROM ColumnDataset d +-- notAField would yield a missing; however (in this case) the missing value is part of the add operation. +-- Thus, the value should evaluate to missing. +-- Previously, this would stuck in infinite loop (See ASTERIXDB-3276). +WHERE d.notAField + 1 > 4 +ORDER BY d.id; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/filter/003/003.007.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/filter/003/003.007.query.sqlpp new file mode 100644 index 0000000..a6af494 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/filter/003/003.007.query.sqlpp @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; +SET `compiler.column.filter` "true"; + +SELECT VALUE item +FROM ColumnDataset d, d.array item +-- notAField would yield a missing; however (in this case) the missing value is part of the add operation. +-- Thus, the value should evaluate to missing. +-- Previously, this would stuck in infinite loop (See ASTERIXDB-3276). +WHERE item.notAField + 1 > 4 +ORDER BY d.id; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/column/filter/003/003.006.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/filter/003/003.006.adm new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/filter/003/003.006.adm diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/column/filter/003/003.007.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/filter/003/003.007.adm new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/filter/003/003.007.adm diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/evaluator/ColumnIterableFilterEvaluatorFactory.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/evaluator/ColumnIterableFilterEvaluatorFactory.java index 1264c24..742511b 100644 --- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/evaluator/ColumnIterableFilterEvaluatorFactory.java +++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/evaluator/ColumnIterableFilterEvaluatorFactory.java @@ -40,8 +40,8 @@ @Override public IColumnIterableFilterEvaluator create(FilterAccessorProvider filterAccessorProvider, IEvaluatorContext context) throws HyracksDataException { - IScalarEvaluator evaluator = evaluatorFactory.createScalarEvaluator(context); List<IColumnValuesReader> readers = filterAccessorProvider.getFilterColumnReaders(); + IScalarEvaluator evaluator = evaluatorFactory.createScalarEvaluator(context); if (readers.stream().anyMatch(IColumnValuesReader::isRepeated)) { return new ColumnarRepeatedIterableFilterEvaluator(evaluator, readers); } diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/visitor/PathExtractorVisitor.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/visitor/PathExtractorVisitor.java index de4df8a..efff918 100644 --- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/visitor/PathExtractorVisitor.java +++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/visitor/PathExtractorVisitor.java @@ -58,9 +58,7 @@ delimiters.clear(); AbstractSchemaNode node = path.accept(this, null); ATypeTag typeTag = node.getTypeTag(); - if (typeTag == ATypeTag.MISSING) { - return Collections.emptyList(); - } else if (typeTag == ATypeTag.UNION) { + if (typeTag == ATypeTag.UNION) { UnionSchemaNode unionNode = (UnionSchemaNode) node; Collection<AbstractSchemaNode> children = unionNode.getChildren().values(); List<IColumnValuesReader> unionReaders = new ArrayList<>(); diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnTupleReference.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnTupleReference.java index 71497f5..ba5ffb7 100644 --- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnTupleReference.java +++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnTupleReference.java @@ -24,6 +24,7 @@ import org.apache.asterix.column.assembler.value.MissingValueGetter; import org.apache.asterix.column.bytes.stream.in.AbstractBytesInputStream; import org.apache.asterix.column.bytes.stream.in.ByteBufferInputStream; +import org.apache.asterix.column.bytes.stream.in.DummyBytesInputStream; import org.apache.asterix.column.bytes.stream.in.MultiByteBufferInputStream; import org.apache.asterix.column.filter.FilterAccessorProvider; import org.apache.asterix.column.filter.IColumnFilterEvaluator; @@ -68,7 +69,10 @@ int numberOfPrimaryKeys = columnMetadata.getNumberOfPrimaryKeys(); filteredColumnStreams = new AbstractBytesInputStream[columnMetadata.getNumberOfFilteredColumns()]; for (int i = 0; i < filteredColumnStreams.length; i++) { - if (filterColumnReaders.get(i).getColumnIndex() >= numberOfPrimaryKeys) { + int columnIndex = filterColumnReaders.get(i).getColumnIndex(); + if (columnIndex < 0) { + filteredColumnStreams[i] = DummyBytesInputStream.INSTANCE; + } else if (columnIndex >= numberOfPrimaryKeys) { filteredColumnStreams[i] = new MultiByteBufferInputStream(); } else { filteredColumnStreams[i] = new ByteBufferInputStream(); diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnWithMetaTupleReference.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnWithMetaTupleReference.java index 798c15f..a2d6b54 100644 --- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnWithMetaTupleReference.java +++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnWithMetaTupleReference.java @@ -24,6 +24,7 @@ import org.apache.asterix.column.assembler.value.MissingValueGetter; import org.apache.asterix.column.bytes.stream.in.AbstractBytesInputStream; import org.apache.asterix.column.bytes.stream.in.ByteBufferInputStream; +import org.apache.asterix.column.bytes.stream.in.DummyBytesInputStream; import org.apache.asterix.column.bytes.stream.in.MultiByteBufferInputStream; import org.apache.asterix.column.filter.FilterAccessorProvider; import org.apache.asterix.column.filter.IColumnFilterEvaluator; @@ -71,7 +72,10 @@ int numberOfPrimaryKeys = columnMetadata.getNumberOfPrimaryKeys(); filteredColumnStreams = new AbstractBytesInputStream[columnMetadata.getNumberOfFilteredColumns()]; for (int i = 0; i < filteredColumnStreams.length; i++) { - if (filterColumnReaders.get(i).getColumnIndex() >= numberOfPrimaryKeys) { + int columnIndex = filterColumnReaders.get(i).getColumnIndex(); + if (columnIndex < 0) { + filteredColumnStreams[i] = DummyBytesInputStream.INSTANCE; + } else if (columnIndex >= numberOfPrimaryKeys) { filteredColumnStreams[i] = new MultiByteBufferInputStream(); } else { filteredColumnStreams[i] = new ByteBufferInputStream(); diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java index 021b9f5..8ed6df6 100644 --- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java +++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java @@ -57,18 +57,19 @@ AbstractColumnValuesReader(AbstractValueReader valueReader, int columnIndex, int maxLevel, boolean primaryKey) { this.valueReader = valueReader; this.columnIndex = columnIndex; - this.maxLevel = maxLevel; + this.maxLevel = valueReader.getTypeTag() == ATypeTag.MISSING ? Integer.MAX_VALUE : maxLevel; definitionLevels = new ParquetRunLengthBitPackingHybridDecoder(ColumnValuesUtil.getBitWidth(maxLevel)); valuesStream = primaryKey ? new ByteBufferInputStream() : new MultiByteBufferInputStream(); this.primaryKey = primaryKey; } final void nextLevel() throws HyracksDataException { + valueIndex++; + if (allMissing) { return; } - valueIndex++; int actualLevel = definitionLevels.readInt(); //Check whether the level is for a null value nullLevel = ColumnValuesUtil.isNull(nullBitMask, actualLevel); diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/value/NoOpValueReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/value/NoOpValueReader.java index 1982c54..0d59499 100644 --- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/value/NoOpValueReader.java +++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/value/NoOpValueReader.java @@ -24,7 +24,7 @@ import org.apache.asterix.om.types.ATypeTag; import org.apache.hyracks.api.exceptions.HyracksDataException; -public class NoOpValueReader extends AbstractValueReader { +public final class NoOpValueReader extends AbstractValueReader { public static final AbstractValueReader INSTANCE = new NoOpValueReader(); private NoOpValueReader() { @@ -42,7 +42,7 @@ @Override public ATypeTag getTypeTag() { - throw new UnsupportedOperationException(getClass().getName()); + return ATypeTag.MISSING; } @Override diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/AbstractColumnTupleReference.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/AbstractColumnTupleReference.java index 0ac8c75..283522a 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/AbstractColumnTupleReference.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/AbstractColumnTupleReference.java @@ -68,7 +68,9 @@ filterBufferProviders = new IColumnBufferProvider[numberOfFilteredColumns]; for (int i = 0; i < numberOfFilteredColumns; i++) { int columnIndex = info.getFilteredColumnIndex(i); - if (columnIndex >= numberOfPrimaryKeys) { + if (columnIndex < 0) { + filterBufferProviders[i] = DummyColumnBufferProvider.INSTANCE; + } else if (columnIndex >= numberOfPrimaryKeys) { filterBufferProviders[i] = new ColumnMultiBufferProvider(columnIndex, multiPageOp); } else { filterBufferProviders[i] = new ColumnSingleBufferProvider(columnIndex); -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17849 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: Ib32c9a1fe2814da4091c96e504d694b67d0d0d56 Gerrit-Change-Number: 17849 Gerrit-PatchSet: 1 Gerrit-Owner: Wail Alkowaileet <[email protected]> Gerrit-MessageType: newchange
