This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 69f382ec3d00466284e9f4b9fabfa3c60cbbd5fd Author: Wail Alkowaileet <[email protected]> AuthorDate: Fri Aug 16 09:47:41 2024 +0300 [ASTERIXDB-3482][STO] Fix handling NULLs - user model changes: no - storage format changes: yes - interface changes: no Details: - Ensure to write the nullbit (the level MSB) on merge - Fix the ClassCastException in union node when the originalType is a nested node Ext-ref: MB-63167 Change-Id: I19321e6e1cd2d569ba989afc45897da7054b86bd Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18658 Integration-Tests: Jenkins <[email protected]> Reviewed-by: Murtadha Hubail <[email protected]> Tested-by: Wail Alkowaileet <[email protected]> --- .../missing-null-values/007/006.001.ddl.sqlpp | 28 ++++++++++++ .../missing-null-values/007/006.002.update.sqlpp | 53 ++++++++++++++++++++++ .../missing-null-values/007/006.003.query.sqlpp | 24 ++++++++++ .../missing-null-values/008/006.001.ddl.sqlpp | 28 ++++++++++++ .../missing-null-values/008/006.002.update.sqlpp | 53 ++++++++++++++++++++++ .../missing-null-values/008/006.003.query.sqlpp | 24 ++++++++++ .../column/missing-null-values/007/007.003.adm | 24 ++++++++++ .../column/missing-null-values/008/008.003.adm | 24 ++++++++++ .../src/test/resources/runtimets/sqlpp_queries.xml | 10 ++++ .../operation/lsm/flush/ColumnTransformer.java | 13 ++++-- .../values/reader/AbstractColumnValuesReader.java | 9 ++++ .../values/reader/PrimitiveColumnValuesReader.java | 2 +- .../RepeatedPrimitiveColumnValuesReader.java | 5 +- .../data/211-unionArrayPrimitiveNull.json | 3 ++ .../assembler/211-unionArrayPrimitiveNull.json | 3 ++ .../result/small/211-unionArrayPrimitiveNull.json | 3 ++ .../transformer/211-unionArrayPrimitiveNull.schema | 7 +++ 17 files changed, 306 insertions(+), 7 deletions(-) diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/missing-null-values/007/006.001.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/missing-null-values/007/006.001.ddl.sqlpp new file mode 100644 index 0000000000..064d705a52 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/missing-null-values/007/006.001.ddl.sqlpp @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +DROP DATAVERSE test IF EXISTS; +CREATE DATAVERSE test; + +USE test; + +CREATE DATASET ColumnDataset +PRIMARY KEY (id: int) WITH { + "storage-format": {"format" : "column"} +}; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/missing-null-values/007/006.002.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/missing-null-values/007/006.002.update.sqlpp new file mode 100644 index 0000000000..205b6a2658 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/missing-null-values/007/006.002.update.sqlpp @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; + +UPSERT INTO ColumnDataset ( + {"id":0, "a": [1]}, + {"id":1, "a": [1]}, + {"id":2, "a": [1]}, + {"id":3, "a": [1]}, + {"id":4, "a": [1]}, + {"id":5, "a": [1]}, + {"id":6, "a": [1]}, + {"id":7, "a": [1]} +); + +UPSERT INTO ColumnDataset ( + {"id":8, "a": 5}, + {"id":9, "a": 5}, + {"id":10, "a": 5}, + {"id":11, "a": 5}, + {"id":12, "a": 5}, + {"id":13, "a": 5}, + {"id":14, "a": 5}, + {"id":15, "a": 5} +); + +UPSERT INTO ColumnDataset ( + {"id":16, "a": null}, + {"id":17, "a": null}, + {"id":18, "a": null}, + {"id":19, "a": null}, + {"id":20, "a": null}, + {"id":21, "a": null}, + {"id":22, "a": null}, + {"id":23, "a": null} +); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/missing-null-values/007/006.003.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/missing-null-values/007/006.003.query.sqlpp new file mode 100644 index 0000000000..05d4c05212 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/missing-null-values/007/006.003.query.sqlpp @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; + +SELECT p.a, p.a IS NULL AS null_check +FROM ColumnDataset p +ORDER BY p.id diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/missing-null-values/008/006.001.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/missing-null-values/008/006.001.ddl.sqlpp new file mode 100644 index 0000000000..064d705a52 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/missing-null-values/008/006.001.ddl.sqlpp @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +DROP DATAVERSE test IF EXISTS; +CREATE DATAVERSE test; + +USE test; + +CREATE DATASET ColumnDataset +PRIMARY KEY (id: int) WITH { + "storage-format": {"format" : "column"} +}; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/missing-null-values/008/006.002.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/missing-null-values/008/006.002.update.sqlpp new file mode 100644 index 0000000000..e1cbb81aac --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/missing-null-values/008/006.002.update.sqlpp @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; + +UPSERT INTO ColumnDataset ( + {"id":0, "a": [1]}, + {"id":1, "a": [1]}, + {"id":2, "a": [1]}, + {"id":3, "a": [1]}, + {"id":4, "a": [1]}, + {"id":5, "a": [1]}, + {"id":6, "a": [1]}, + {"id":7, "a": [1]} +); + +UPSERT INTO ColumnDataset ( + {"id":8, "a": [5]}, + {"id":9, "a": [5]}, + {"id":10, "a": [5]}, + {"id":11, "a": [5]}, + {"id":12, "a": [5]}, + {"id":13, "a": [5]}, + {"id":14, "a": [5]}, + {"id":15, "a": [5]} +); + +UPSERT INTO ColumnDataset ( + {"id":16, "a": null}, + {"id":17, "a": null}, + {"id":18, "a": null}, + {"id":19, "a": null}, + {"id":20, "a": null}, + {"id":21, "a": null}, + {"id":22, "a": null}, + {"id":23, "a": null} +); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/missing-null-values/008/006.003.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/missing-null-values/008/006.003.query.sqlpp new file mode 100644 index 0000000000..05d4c05212 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/missing-null-values/008/006.003.query.sqlpp @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; + +SELECT p.a, p.a IS NULL AS null_check +FROM ColumnDataset p +ORDER BY p.id diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/column/missing-null-values/007/007.003.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/missing-null-values/007/007.003.adm new file mode 100644 index 0000000000..4d2d64318f --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/missing-null-values/007/007.003.adm @@ -0,0 +1,24 @@ +{ "null_check": false, "a": [ 1 ] } +{ "null_check": false, "a": [ 1 ] } +{ "null_check": false, "a": [ 1 ] } +{ "null_check": false, "a": [ 1 ] } +{ "null_check": false, "a": [ 1 ] } +{ "null_check": false, "a": [ 1 ] } +{ "null_check": false, "a": [ 1 ] } +{ "null_check": false, "a": [ 1 ] } +{ "null_check": false, "a": 5 } +{ "null_check": false, "a": 5 } +{ "null_check": false, "a": 5 } +{ "null_check": false, "a": 5 } +{ "null_check": false, "a": 5 } +{ "null_check": false, "a": 5 } +{ "null_check": false, "a": 5 } +{ "null_check": false, "a": 5 } +{ "null_check": true, "a": null } +{ "null_check": true, "a": null } +{ "null_check": true, "a": null } +{ "null_check": true, "a": null } +{ "null_check": true, "a": null } +{ "null_check": true, "a": null } +{ "null_check": true, "a": null } +{ "null_check": true, "a": null } diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/column/missing-null-values/008/008.003.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/missing-null-values/008/008.003.adm new file mode 100644 index 0000000000..a1c54ffcce --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/missing-null-values/008/008.003.adm @@ -0,0 +1,24 @@ +{ "null_check": false, "a": [ 1 ] } +{ "null_check": false, "a": [ 1 ] } +{ "null_check": false, "a": [ 1 ] } +{ "null_check": false, "a": [ 1 ] } +{ "null_check": false, "a": [ 1 ] } +{ "null_check": false, "a": [ 1 ] } +{ "null_check": false, "a": [ 1 ] } +{ "null_check": false, "a": [ 1 ] } +{ "null_check": false, "a": [ 5 ] } +{ "null_check": false, "a": [ 5 ] } +{ "null_check": false, "a": [ 5 ] } +{ "null_check": false, "a": [ 5 ] } +{ "null_check": false, "a": [ 5 ] } +{ "null_check": false, "a": [ 5 ] } +{ "null_check": false, "a": [ 5 ] } +{ "null_check": false, "a": [ 5 ] } +{ "null_check": true, "a": null } +{ "null_check": true, "a": null } +{ "null_check": true, "a": null } +{ "null_check": true, "a": null } +{ "null_check": true, "a": null } +{ "null_check": true, "a": null } +{ "null_check": true, "a": null } +{ "null_check": true, "a": null } diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml index d139bd0868..2aba75859b 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml @@ -16368,6 +16368,16 @@ <output-dir compare="Text">missing-null-values/006</output-dir> </compilation-unit> </test-case> + <test-case FilePath="column"> + <compilation-unit name="missing-null-values/007"> + <output-dir compare="Text">missing-null-values/007</output-dir> + </compilation-unit> + </test-case> + <test-case FilePath="column"> + <compilation-unit name="missing-null-values/008"> + <output-dir compare="Text">missing-null-values/008</output-dir> + </compilation-unit> + </test-case> <test-case FilePath="column"> <compilation-unit name="empty-array/001"> <output-dir compare="Text">empty-array/001</output-dir> diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/ColumnTransformer.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/ColumnTransformer.java index cccac50290..71b561a7c8 100644 --- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/ColumnTransformer.java +++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/ColumnTransformer.java @@ -190,13 +190,18 @@ public class ColumnTransformer implements ILazyVisitablePointableVisitor<Abstrac currentParent = unionNode; ATypeTag childTypeTag = pointable.getTypeTag(); - AbstractSchemaNode actualNode; + if (childTypeTag == ATypeTag.NULL || childTypeTag == ATypeTag.MISSING) { - actualNode = unionNode.getOriginalType(); + /* + * NULL and MISSING are tracked since the start to be written in the originalType (i.e., the type + * before injecting a union between the parent and the original node). + */ + AbstractSchemaNode actualNode = unionNode.getOriginalType(); + acceptActualNode(pointable, actualNode); } else { - actualNode = unionNode.getOrCreateChild(pointable.getTypeTag(), columnMetadata); + AbstractSchemaNode actualNode = unionNode.getOrCreateChild(pointable.getTypeTag(), columnMetadata); + pointable.accept(this, actualNode); } - pointable.accept(this, actualNode); currentParent = previousParent; columnMetadata.exitNode(node); diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java index b03da57b32..0942a23ca3 100644 --- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java +++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java @@ -202,6 +202,15 @@ abstract class AbstractColumnValuesReader implements IColumnValuesReader { } } + protected final void writeLevel(IColumnValuesWriter writer) throws HyracksDataException { + if (isNull()) { + // This will prepend the nullBitMask + writer.writeNull(level); + } else { + writer.writeLevel(level); + } + } + protected void appendCommon(ObjectNode node) { node.put("typeTag", getTypeTag().toString()); node.put("columnIndex", columnIndex); diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/PrimitiveColumnValuesReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/PrimitiveColumnValuesReader.java index f1c29293d8..7b02c70726 100644 --- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/PrimitiveColumnValuesReader.java +++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/PrimitiveColumnValuesReader.java @@ -100,7 +100,7 @@ public final class PrimitiveColumnValuesReader extends AbstractColumnValuesReade throw e; } - writer.writeLevel(level); + writeLevel(writer); if (primaryKey || isValue()) { try { writer.writeValue(this); diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/RepeatedPrimitiveColumnValuesReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/RepeatedPrimitiveColumnValuesReader.java index 3f90a4baaa..0f3b817e9e 100644 --- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/RepeatedPrimitiveColumnValuesReader.java +++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/RepeatedPrimitiveColumnValuesReader.java @@ -99,15 +99,16 @@ public final class RepeatedPrimitiveColumnValuesReader extends AbstractColumnVal if (isRepeatedValue()) { while (!isLastDelimiter()) { - writer.writeLevel(level); + writeLevel(writer); if (isValue()) { writer.writeValue(this); } doNextAndCheck(); } } + //Add last delimiter, or NULL/MISSING - writer.writeLevel(level); + writeLevel(writer); } @Override diff --git a/asterixdb/asterix-column/src/test/resources/data/211-unionArrayPrimitiveNull.json b/asterixdb/asterix-column/src/test/resources/data/211-unionArrayPrimitiveNull.json new file mode 100644 index 0000000000..041afdc619 --- /dev/null +++ b/asterixdb/asterix-column/src/test/resources/data/211-unionArrayPrimitiveNull.json @@ -0,0 +1,3 @@ +{"a": [0]} +{"a": 1} +{"a": null} diff --git a/asterixdb/asterix-column/src/test/resources/result/assembler/211-unionArrayPrimitiveNull.json b/asterixdb/asterix-column/src/test/resources/result/assembler/211-unionArrayPrimitiveNull.json new file mode 100644 index 0000000000..1b928a2910 --- /dev/null +++ b/asterixdb/asterix-column/src/test/resources/result/assembler/211-unionArrayPrimitiveNull.json @@ -0,0 +1,3 @@ +{"a":[0]} +{"a":1} +{"a":null} diff --git a/asterixdb/asterix-column/src/test/resources/result/small/211-unionArrayPrimitiveNull.json b/asterixdb/asterix-column/src/test/resources/result/small/211-unionArrayPrimitiveNull.json new file mode 100644 index 0000000000..1b928a2910 --- /dev/null +++ b/asterixdb/asterix-column/src/test/resources/result/small/211-unionArrayPrimitiveNull.json @@ -0,0 +1,3 @@ +{"a":[0]} +{"a":1} +{"a":null} diff --git a/asterixdb/asterix-column/src/test/resources/result/transformer/211-unionArrayPrimitiveNull.schema b/asterixdb/asterix-column/src/test/resources/result/transformer/211-unionArrayPrimitiveNull.schema new file mode 100644 index 0000000000..8fc4a85f39 --- /dev/null +++ b/asterixdb/asterix-column/src/test/resources/result/transformer/211-unionArrayPrimitiveNull.schema @@ -0,0 +1,7 @@ +root +|-- a: union <level: 1> +| |-- bigint: bigint <level: 1, index: 1> +| | |-- Def size: 3 [(0,1),(1,1),(0,1)] +| |-- array: array <level: 1> +| | |-- item: bigint <level: 2, index: 0> +| | | |-- Def size: 5 [(2,1),(1,1),(0,2),(4,1)]
