This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit a013fb7a89580a1c90f82e6d01dc0078ac4bcd93 Author: Peeyush Gupta <[email protected]> AuthorDate: Thu Jun 5 16:09:36 2025 -0700 [ASTERIXDB-3620][STO] Fix writing null values to columnar storage - user model changes: no - storage format changes: no - interface changes: no Ext-ref: MB-67051 Change-Id: I32f8c19c80c5ac898781a4931a30459b3a801e4d Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19873 Reviewed-by: Ritik Raj <[email protected]> Tested-by: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> --- .../missing-null-values/009/009.001.ddl.sqlpp | 33 ++++++++ .../missing-null-values/009/009.002.update.sqlpp | 90 ++++++++++++++++++++++ .../missing-null-values/009/009.003.query.sqlpp | 24 ++++++ .../missing-null-values/009/009.004.query.sqlpp | 24 ++++++ .../column/missing-null-values/009/009.003.adm | 4 + .../column/missing-null-values/009/009.004.adm | 4 + .../runtimets/testsuite_single_partition_sqlpp.xml | 5 ++ .../operation/lsm/flush/BatchFinalizerVisitor.java | 4 +- 8 files changed, 186 insertions(+), 2 deletions(-) diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/missing-null-values/009/009.001.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/missing-null-values/009/009.001.ddl.sqlpp new file mode 100644 index 0000000000..d3ae41c39c --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/missing-null-values/009/009.001.ddl.sqlpp @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +DROP DATAVERSE test IF EXISTS; +CREATE DATAVERSE test; + +USE test; + +CREATE DATASET ColumnDataset1 +PRIMARY KEY (id: string) WITH { + "storage-format": {"format" : "column"} +}; + +CREATE DATASET ColumnDataset2 +PRIMARY KEY (id: string) WITH { + "storage-format": {"format" : "column"} +}; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/missing-null-values/009/009.002.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/missing-null-values/009/009.002.update.sqlpp new file mode 100644 index 0000000000..80eee692eb --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/missing-null-values/009/009.002.update.sqlpp @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; + +INSERT INTO ColumnDataset1 +[ + { + "id": "2489669963", + "payload": { + "issue": { + "milestone": { + "creator": { + "avatar_url": "https://avatars.githubusercontent.com/u/6339799?v=3"; + } + } + } + } + }, + { + "id": "2489669966", + "payload": { + "issue": NULL + } + }, + { + "id": "2489669970", + "payload": { + "issue": NULL + } + }, + { + "id": "2489669984", + "payload": { + "issue": { + "milestone": NULL + } + } + } +]; + +INSERT INTO ColumnDataset2 +[ + { + "id": "2489669963", + "payload": { + "issue": { + "milestone": { + "creator": ["a", "b", "c"] + } + } + } + }, + { + "id": "2489669966", + "payload": { + "issue": null + } + }, + { + "id": "2489669970", + "payload": { + "issue": null + } + }, + { + "id": "2489669984", + "payload": { + "issue": { + "milestone": null + } + } + } +] \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/missing-null-values/009/009.003.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/missing-null-values/009/009.003.query.sqlpp new file mode 100644 index 0000000000..b29d752c08 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/missing-null-values/009/009.003.query.sqlpp @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; + +SELECT value v +FROM ColumnDataset1 v +ORDER BY v.id diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/missing-null-values/009/009.004.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/missing-null-values/009/009.004.query.sqlpp new file mode 100644 index 0000000000..a323d08e1f --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/missing-null-values/009/009.004.query.sqlpp @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; + +SELECT value v +FROM ColumnDataset2 v +ORDER BY v.id \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/column/missing-null-values/009/009.003.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/missing-null-values/009/009.003.adm new file mode 100644 index 0000000000..9aaf739662 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/missing-null-values/009/009.003.adm @@ -0,0 +1,4 @@ +{ "id": "2489669963", "payload": { "issue": { "milestone": { "creator": { "avatar_url": "https://avatars.githubusercontent.com/u/6339799?v=3"; } } } } } +{ "id": "2489669966", "payload": { "issue": null } } +{ "id": "2489669970", "payload": { "issue": null } } +{ "id": "2489669984", "payload": { "issue": { "milestone": null } } } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/column/missing-null-values/009/009.004.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/missing-null-values/009/009.004.adm new file mode 100644 index 0000000000..5cc8f60eb3 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/missing-null-values/009/009.004.adm @@ -0,0 +1,4 @@ +{ "id": "2489669963", "payload": { "issue": { "milestone": { "creator": [ "a", "b", "c" ] } } } } +{ "id": "2489669966", "payload": { "issue": null } } +{ "id": "2489669970", "payload": { "issue": null } } +{ "id": "2489669984", "payload": { "issue": { "milestone": null } } } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_single_partition_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_single_partition_sqlpp.xml index 8ebcb75b97..5da5ceaddc 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_single_partition_sqlpp.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_single_partition_sqlpp.xml @@ -204,6 +204,11 @@ <output-dir compare="Text">missing-null-values/008</output-dir> </compilation-unit> </test-case> + <test-case FilePath="column"> + <compilation-unit name="missing-null-values/009"> + <output-dir compare="Text">missing-null-values/009</output-dir> + </compilation-unit> + </test-case> <test-case FilePath="column"> <compilation-unit name="empty-array/001"> <output-dir compare="Text">empty-array/001</output-dir> diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/BatchFinalizerVisitor.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/BatchFinalizerVisitor.java index 951a9fe7e6..e3407a12b5 100644 --- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/BatchFinalizerVisitor.java +++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/BatchFinalizerVisitor.java @@ -66,7 +66,7 @@ public final class BatchFinalizerVisitor implements ISchemaNodeVisitor<Void, Abs @Override public Void visit(ObjectSchemaNode objectNode, AbstractSchemaNestedNode arg) throws HyracksDataException { level++; - columnSchemaMetadata.flushDefinitionLevels(level, arg, objectNode); + columnSchemaMetadata.flushDefinitionLevels(level - 1, arg, objectNode); List<AbstractSchemaNode> children = objectNode.getChildren(); for (int i = 0; i < children.size(); i++) { children.get(i).accept(this, objectNode); @@ -81,7 +81,7 @@ public final class BatchFinalizerVisitor implements ISchemaNodeVisitor<Void, Abs public Void visit(AbstractCollectionSchemaNode collectionNode, AbstractSchemaNestedNode arg) throws HyracksDataException { level++; - columnSchemaMetadata.flushDefinitionLevels(level, arg, collectionNode); + columnSchemaMetadata.flushDefinitionLevels(level - 1, arg, collectionNode); collectionNode.getItemNode().accept(this, collectionNode); collectionNode.setCounter(0); columnSchemaMetadata.clearDefinitionLevels(collectionNode);
