This is an automated email from the ASF dual-hosted git repository. mhubail pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push: new bacbacbfe7 [ASTERIXDB-3166][FUN] Error with get_object_fields with open list type bacbacbfe7 is described below commit bacbacbfe77fdd7e3a1e8225834826cb0b6d97c7 Author: Peeyush Gupta <peeyush.gu...@couchbase.com> AuthorDate: Thu Apr 13 15:49:00 2023 -0700 [ASTERIXDB-3166][FUN] Error with get_object_fields with open list type - user model changes: no - storage format changes: no - interface changes: no Details: get_object_fields function causes null pointer exception when used on a dataset containing an open list type. Change-Id: Ie413d286738f571df81e2f450a3fb0b132dc5521 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17488 Reviewed-by: Peeyush Gupta <peeyush.gu...@couchbase.com> Reviewed-by: Ali Alsuliman <ali.al.solai...@gmail.com> Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu> --- .../queries_sqlpp/objects/ObjectsQueries.xml | 5 +++ .../open-list-type/open-list-type.1.ddl.sqlpp | 34 +++++++++++++++ .../open-list-type/open-list-type.2.update.sqlpp | 34 +++++++++++++++ .../open-list-type/open-list-type.3.query.sqlpp | 22 ++++++++++ .../open-list-type/open-list-type.4.query.sqlpp | 22 ++++++++++ .../open-list-type/open-list-type.3.adm | 1 + .../open-list-type/open-list-type.4.adm | 1 + .../om/pointables/nonvisitor/AListPointable.java | 2 +- .../functions/records/RecordFieldsUtil.java | 51 ++++++++++++++-------- 9 files changed, 152 insertions(+), 20 deletions(-) diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml index 77fc7c5682..1756378f84 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml @@ -82,6 +82,11 @@ <output-dir compare="Text">missing-fields</output-dir> </compilation-unit> </test-case> + <test-case FilePath="objects/get-object-fields"> + <compilation-unit name="open-list-type"> + <output-dir compare="Text">open-list-type</output-dir> + </compilation-unit> + </test-case> <test-case FilePath="objects/get-object-field-value"> <compilation-unit name="documentation-example"> <output-dir compare="Text">documentation-example</output-dir> diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.1.ddl.sqlpp new file mode 100644 index 0000000000..402cc11311 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.1.ddl.sqlpp @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Testing get-object-fields where some schema (complex) fields are missing or null + */ + +DROP DATAVERSE test IF EXISTS; +CREATE DATAVERSE test; + +USE test; + +CREATE TYPE t1 AS { + id: string +}; + + +CREATE DATASET ds1(t1) PRIMARY KEY id; +CREATE DATASET ds2(t1) PRIMARY KEY id; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.2.update.sqlpp new file mode 100644 index 0000000000..d5e98b4992 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.2.update.sqlpp @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; + +INSERT INTO ds1([ +{"id": "xyz", +"some_record": {"x1": 9, "nested_list": ["y1", "y2"], "nested_rec": {"f1": 7}}, +"some_list2": [[1,2],[3,4]], +"some_list3": [{"a": 5}, {"b": 6}] +}]); + +INSERT INTO ds2([ +{"id": "xyz", +"some_record": {"x1": 9, "nested_list": {{"y1", "y2"}}, "nested_rec": {"f1": 7}}, +"some_list2": {{[1,2],{{3,4}}}}, +"some_list3": {{{"a": 5}, {"b": 6}}} +}]); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.3.query.sqlpp new file mode 100644 index 0000000000..d45f0834b2 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.3.query.sqlpp @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; + +SELECT VALUE get_object_fields(d) FROM ds1 d; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.4.query.sqlpp new file mode 100644 index 0000000000..5556e29222 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.4.query.sqlpp @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; + +SELECT VALUE get_object_fields(d) FROM ds2 d; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/open-list-type/open-list-type.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/open-list-type/open-list-type.3.adm new file mode 100644 index 0000000000..e506c070eb --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/open-list-type/open-list-type.3.adm @@ -0,0 +1 @@ +[ { "field-name": "id", "field-type": "string", "is-open": false }, { "field-name": "some_record", "field-type": "object", "is-open": true, "nested": [ { "field-name": "x1", "field-type": "bigint", "is-open": true }, { "field-name": "nested_rec", "field-type": "object", "is-open": true, "nested": [ { "field-name": "f1", "field-type": "bigint", "is-open": true } ] }, { "field-name": "nested_list", "field-type": "array", "is-open": true, "list": [ { "field-type": "string" }, { "field-type" [...] diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/open-list-type/open-list-type.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/open-list-type/open-list-type.4.adm new file mode 100644 index 0000000000..b16e19569e --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/open-list-type/open-list-type.4.adm @@ -0,0 +1 @@ +[ { "field-name": "id", "field-type": "string", "is-open": false }, { "field-name": "some_record", "field-type": "object", "is-open": true, "nested": [ { "field-name": "x1", "field-type": "bigint", "is-open": true }, { "field-name": "nested_rec", "field-type": "object", "is-open": true, "nested": [ { "field-name": "f1", "field-type": "bigint", "is-open": true } ] }, { "field-name": "nested_list", "field-type": "multiset", "is-open": true, "list": [ { "field-type": "string" }, { "field-ty [...] diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/nonvisitor/AListPointable.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/nonvisitor/AListPointable.java index 1e3ba12d3c..20af4dc943 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/nonvisitor/AListPointable.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/nonvisitor/AListPointable.java @@ -190,7 +190,7 @@ public class AListPointable extends AbstractPointable { if (isFixedType(inputType)) { return getFixedLength(inputType); } else { - int nextOffset = (getItemCount() > index + 1) ? getItemOffset(inputType, index + 1) : getLength(); + int nextOffset = (getItemCount() > index + 1) ? getItemOffset(inputType, index + 1) : getLength() - 1; return nextOffset - getItemOffset(inputType, index); } } diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordFieldsUtil.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordFieldsUtil.java index 25dd73e595..d7723f0a99 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordFieldsUtil.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordFieldsUtil.java @@ -37,6 +37,7 @@ import org.apache.asterix.om.pointables.nonvisitor.ARecordPointable; import org.apache.asterix.om.types.AOrderedListType; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.ATypeTag; +import org.apache.asterix.om.types.AUnorderedListType; import org.apache.asterix.om.types.AbstractCollectionType; import org.apache.asterix.om.types.BuiltinType; import org.apache.asterix.om.types.EnumDeserializer; @@ -78,6 +79,9 @@ public class RecordFieldsUtil { SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ABOOLEAN); private final static ARecordType openType = DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE; + private final static AOrderedListType openOrderedListType = DefaultOpenFieldType.NESTED_OPEN_AORDERED_LIST_TYPE; + private final static AUnorderedListType openUnOrderedListType = + DefaultOpenFieldType.NESTED_OPEN_AUNORDERED_LIST_TYPE; public void processRecord(ARecordPointable recordAccessor, ARecordType recType, DataOutput out, int level) throws IOException { @@ -91,7 +95,7 @@ public class RecordFieldsUtil { OrderedListBuilder orderedListBuilder = getOrderedListBuilder(); orderedListBuilder.reset(listType); IARecordBuilder fieldRecordBuilder = getRecordBuilder(); - fieldRecordBuilder.reset(null); + fieldRecordBuilder.reset(openType); int schemeFieldCount = recordAccessor.getSchemeFieldCount(recType); for (int i = 0; i < schemeFieldCount; ++i) { @@ -119,7 +123,7 @@ public class RecordFieldsUtil { tmpValue.reset(); recordAccessor.getClosedFieldValue(recType, i, tmpValue.getDataOutput()); if (tag == ATypeTag.SERIALIZED_RECORD_TYPE_TAG) { - addNestedField(tmpValue, fieldType, fieldRecordBuilder, level + 1); + addRecordField(tmpValue, fieldType, fieldRecordBuilder, level + 1); } else { addListField(tmpValue, fieldType, fieldRecordBuilder, level + 1); } @@ -151,14 +155,15 @@ public class RecordFieldsUtil { // write nested or list types if (tag == ATypeTag.SERIALIZED_RECORD_TYPE_TAG || tag == ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG || tag == ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG) { - IAType fieldType = null; ArrayBackedValueStorage tmpValue = getTempBuffer(); tmpValue.reset(); recordAccessor.getOpenFieldValue(recType, i, tmpValue.getDataOutput()); if (tag == ATypeTag.SERIALIZED_RECORD_TYPE_TAG) { - addNestedField(tmpValue, fieldType, fieldRecordBuilder, level + 1); + addRecordField(tmpValue, openType, fieldRecordBuilder, level + 1); + } else if (tag == ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG) { + addListField(tmpValue, openOrderedListType, fieldRecordBuilder, level + 1); } else { - addListField(tmpValue, fieldType, fieldRecordBuilder, level + 1); + addListField(tmpValue, openUnOrderedListType, fieldRecordBuilder, level + 1); } } @@ -226,7 +231,7 @@ public class RecordFieldsUtil { fieldRecordBuilder.addField(fieldAbvs, valueAbvs); } - public void addNestedField(IValueReference recordArg, IAType fieldType, IARecordBuilder fieldRecordBuilder, + public void addRecordField(IValueReference recordArg, IAType fieldType, IARecordBuilder fieldRecordBuilder, int level) throws IOException { ArrayBackedValueStorage fieldAbvs = getTempBuffer(); ArrayBackedValueStorage valueAbvs = getTempBuffer(); @@ -236,31 +241,26 @@ public class RecordFieldsUtil { stringSerde.serialize(nestedName, fieldAbvs.getDataOutput()); // Value valueAbvs.reset(); - ARecordType newType; - if (fieldType == null) { - newType = openType; - } else { - newType = (ARecordType) fieldType; - } + ARecordType newType = (ARecordType) fieldType; ARecordPointable recordP = getRecordPointable(); recordP.set(recordArg); processRecord(recordP, newType, valueAbvs.getDataOutput(), level); fieldRecordBuilder.addField(fieldAbvs, valueAbvs); } - public void processListValue(IValueReference listArg, IAType fieldType, DataOutput out, int level) + public void processListValue(IValueReference listArg, IAType listType, DataOutput out, int level) throws IOException { ArrayBackedValueStorage itemValue = getTempBuffer(); IARecordBuilder listRecordBuilder = getRecordBuilder(); - AListPointable list = getListPointable(fieldType.getTypeTag()); + AListPointable list = getListPointable(listType.getTypeTag()); list.set(listArg); OrderedListBuilder innerListBuilder = getOrderedListBuilder(); - innerListBuilder.reset(listType); + innerListBuilder.reset(RecordFieldsUtil.listType); - listRecordBuilder.reset(null); - AbstractCollectionType act = (AbstractCollectionType) fieldType; + listRecordBuilder.reset(openType); + AbstractCollectionType act = (AbstractCollectionType) listType; int itemCount = list.getItemCount(); for (int l = 0; l < itemCount; l++) { itemValue.reset(); @@ -269,11 +269,24 @@ public class RecordFieldsUtil { byte tagId = list.getItemTag(act, l); addFieldType(tagId, listRecordBuilder); - if (tagId == ATypeTag.SERIALIZED_RECORD_TYPE_TAG) { + if (tagId == ATypeTag.SERIALIZED_RECORD_TYPE_TAG || tagId == ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG + || tagId == ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG) { ArrayBackedValueStorage tmpAbvs = getTempBuffer(); tmpAbvs.reset(); list.getItemValue(act, l, tmpAbvs.getDataOutput()); - addNestedField(tmpAbvs, act.getItemType(), listRecordBuilder, level + 1); + IAType itemType; + + if (tagId == ATypeTag.SERIALIZED_RECORD_TYPE_TAG) { + itemType = act.getItemType().getTypeTag() != ATypeTag.ANY ? act.getItemType() : openType; + addRecordField(tmpAbvs, itemType, listRecordBuilder, level + 1); + } else if (tagId == ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG) { + itemType = act.getItemType().getTypeTag() != ATypeTag.ANY ? act.getItemType() : openOrderedListType; + addListField(tmpAbvs, itemType, listRecordBuilder, level + 1); + } else { + itemType = + act.getItemType().getTypeTag() != ATypeTag.ANY ? act.getItemType() : openUnOrderedListType; + addListField(tmpAbvs, itemType, listRecordBuilder, level + 1); + } } listRecordBuilder.write(itemValue.getDataOutput(), true);