This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new bacbacbfe7 [ASTERIXDB-3166][FUN] Error with get_object_fields with
open list type
bacbacbfe7 is described below
commit bacbacbfe77fdd7e3a1e8225834826cb0b6d97c7
Author: Peeyush Gupta <[email protected]>
AuthorDate: Thu Apr 13 15:49:00 2023 -0700
[ASTERIXDB-3166][FUN] Error with get_object_fields with open list type
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
get_object_fields function causes null pointer exception when used
on a dataset containing an open list type.
Change-Id: Ie413d286738f571df81e2f450a3fb0b132dc5521
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17488
Reviewed-by: Peeyush Gupta <[email protected]>
Reviewed-by: Ali Alsuliman <[email protected]>
Integration-Tests: Jenkins <[email protected]>
Tested-by: Jenkins <[email protected]>
---
.../queries_sqlpp/objects/ObjectsQueries.xml | 5 +++
.../open-list-type/open-list-type.1.ddl.sqlpp | 34 +++++++++++++++
.../open-list-type/open-list-type.2.update.sqlpp | 34 +++++++++++++++
.../open-list-type/open-list-type.3.query.sqlpp | 22 ++++++++++
.../open-list-type/open-list-type.4.query.sqlpp | 22 ++++++++++
.../open-list-type/open-list-type.3.adm | 1 +
.../open-list-type/open-list-type.4.adm | 1 +
.../om/pointables/nonvisitor/AListPointable.java | 2 +-
.../functions/records/RecordFieldsUtil.java | 51 ++++++++++++++--------
9 files changed, 152 insertions(+), 20 deletions(-)
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml
index 77fc7c5682..1756378f84 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml
@@ -82,6 +82,11 @@
<output-dir compare="Text">missing-fields</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="objects/get-object-fields">
+ <compilation-unit name="open-list-type">
+ <output-dir compare="Text">open-list-type</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="objects/get-object-field-value">
<compilation-unit name="documentation-example">
<output-dir compare="Text">documentation-example</output-dir>
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.1.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.1.ddl.sqlpp
new file mode 100644
index 0000000000..402cc11311
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.1.ddl.sqlpp
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Testing get-object-fields where some schema (complex) fields
are missing or null
+ */
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+
+USE test;
+
+CREATE TYPE t1 AS {
+ id: string
+};
+
+
+CREATE DATASET ds1(t1) PRIMARY KEY id;
+CREATE DATASET ds2(t1) PRIMARY KEY id;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.2.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.2.update.sqlpp
new file mode 100644
index 0000000000..d5e98b4992
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.2.update.sqlpp
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+INSERT INTO ds1([
+{"id": "xyz",
+"some_record": {"x1": 9, "nested_list": ["y1", "y2"], "nested_rec": {"f1": 7}},
+"some_list2": [[1,2],[3,4]],
+"some_list3": [{"a": 5}, {"b": 6}]
+}]);
+
+INSERT INTO ds2([
+{"id": "xyz",
+"some_record": {"x1": 9, "nested_list": {{"y1", "y2"}}, "nested_rec": {"f1":
7}},
+"some_list2": {{[1,2],{{3,4}}}},
+"some_list3": {{{"a": 5}, {"b": 6}}}
+}]);
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.3.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.3.query.sqlpp
new file mode 100644
index 0000000000..d45f0834b2
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.3.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SELECT VALUE get_object_fields(d) FROM ds1 d;
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.4.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.4.query.sqlpp
new file mode 100644
index 0000000000..5556e29222
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.4.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SELECT VALUE get_object_fields(d) FROM ds2 d;
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/open-list-type/open-list-type.3.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/open-list-type/open-list-type.3.adm
new file mode 100644
index 0000000000..e506c070eb
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/open-list-type/open-list-type.3.adm
@@ -0,0 +1 @@
+[ { "field-name": "id", "field-type": "string", "is-open": false }, {
"field-name": "some_record", "field-type": "object", "is-open": true, "nested":
[ { "field-name": "x1", "field-type": "bigint", "is-open": true }, {
"field-name": "nested_rec", "field-type": "object", "is-open": true, "nested":
[ { "field-name": "f1", "field-type": "bigint", "is-open": true } ] }, {
"field-name": "nested_list", "field-type": "array", "is-open": true, "list": [
{ "field-type": "string" }, { "field-type" [...]
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/open-list-type/open-list-type.4.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/open-list-type/open-list-type.4.adm
new file mode 100644
index 0000000000..b16e19569e
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/open-list-type/open-list-type.4.adm
@@ -0,0 +1 @@
+[ { "field-name": "id", "field-type": "string", "is-open": false }, {
"field-name": "some_record", "field-type": "object", "is-open": true, "nested":
[ { "field-name": "x1", "field-type": "bigint", "is-open": true }, {
"field-name": "nested_rec", "field-type": "object", "is-open": true, "nested":
[ { "field-name": "f1", "field-type": "bigint", "is-open": true } ] }, {
"field-name": "nested_list", "field-type": "multiset", "is-open": true, "list":
[ { "field-type": "string" }, { "field-ty [...]
diff --git
a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/nonvisitor/AListPointable.java
b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/nonvisitor/AListPointable.java
index 1e3ba12d3c..20af4dc943 100644
---
a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/nonvisitor/AListPointable.java
+++
b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/nonvisitor/AListPointable.java
@@ -190,7 +190,7 @@ public class AListPointable extends AbstractPointable {
if (isFixedType(inputType)) {
return getFixedLength(inputType);
} else {
- int nextOffset = (getItemCount() > index + 1) ?
getItemOffset(inputType, index + 1) : getLength();
+ int nextOffset = (getItemCount() > index + 1) ?
getItemOffset(inputType, index + 1) : getLength() - 1;
return nextOffset - getItemOffset(inputType, index);
}
}
diff --git
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordFieldsUtil.java
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordFieldsUtil.java
index 25dd73e595..d7723f0a99 100644
---
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordFieldsUtil.java
+++
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordFieldsUtil.java
@@ -37,6 +37,7 @@ import
org.apache.asterix.om.pointables.nonvisitor.ARecordPointable;
import org.apache.asterix.om.types.AOrderedListType;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AUnorderedListType;
import org.apache.asterix.om.types.AbstractCollectionType;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.EnumDeserializer;
@@ -78,6 +79,9 @@ public class RecordFieldsUtil {
SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ABOOLEAN);
private final static ARecordType openType =
DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE;
+ private final static AOrderedListType openOrderedListType =
DefaultOpenFieldType.NESTED_OPEN_AORDERED_LIST_TYPE;
+ private final static AUnorderedListType openUnOrderedListType =
+ DefaultOpenFieldType.NESTED_OPEN_AUNORDERED_LIST_TYPE;
public void processRecord(ARecordPointable recordAccessor, ARecordType
recType, DataOutput out, int level)
throws IOException {
@@ -91,7 +95,7 @@ public class RecordFieldsUtil {
OrderedListBuilder orderedListBuilder = getOrderedListBuilder();
orderedListBuilder.reset(listType);
IARecordBuilder fieldRecordBuilder = getRecordBuilder();
- fieldRecordBuilder.reset(null);
+ fieldRecordBuilder.reset(openType);
int schemeFieldCount = recordAccessor.getSchemeFieldCount(recType);
for (int i = 0; i < schemeFieldCount; ++i) {
@@ -119,7 +123,7 @@ public class RecordFieldsUtil {
tmpValue.reset();
recordAccessor.getClosedFieldValue(recType, i,
tmpValue.getDataOutput());
if (tag == ATypeTag.SERIALIZED_RECORD_TYPE_TAG) {
- addNestedField(tmpValue, fieldType,
fieldRecordBuilder, level + 1);
+ addRecordField(tmpValue, fieldType,
fieldRecordBuilder, level + 1);
} else {
addListField(tmpValue, fieldType, fieldRecordBuilder,
level + 1);
}
@@ -151,14 +155,15 @@ public class RecordFieldsUtil {
// write nested or list types
if (tag == ATypeTag.SERIALIZED_RECORD_TYPE_TAG || tag ==
ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG
|| tag == ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG) {
- IAType fieldType = null;
ArrayBackedValueStorage tmpValue = getTempBuffer();
tmpValue.reset();
recordAccessor.getOpenFieldValue(recType, i,
tmpValue.getDataOutput());
if (tag == ATypeTag.SERIALIZED_RECORD_TYPE_TAG) {
- addNestedField(tmpValue, fieldType, fieldRecordBuilder,
level + 1);
+ addRecordField(tmpValue, openType, fieldRecordBuilder,
level + 1);
+ } else if (tag == ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG) {
+ addListField(tmpValue, openOrderedListType,
fieldRecordBuilder, level + 1);
} else {
- addListField(tmpValue, fieldType, fieldRecordBuilder,
level + 1);
+ addListField(tmpValue, openUnOrderedListType,
fieldRecordBuilder, level + 1);
}
}
@@ -226,7 +231,7 @@ public class RecordFieldsUtil {
fieldRecordBuilder.addField(fieldAbvs, valueAbvs);
}
- public void addNestedField(IValueReference recordArg, IAType fieldType,
IARecordBuilder fieldRecordBuilder,
+ public void addRecordField(IValueReference recordArg, IAType fieldType,
IARecordBuilder fieldRecordBuilder,
int level) throws IOException {
ArrayBackedValueStorage fieldAbvs = getTempBuffer();
ArrayBackedValueStorage valueAbvs = getTempBuffer();
@@ -236,31 +241,26 @@ public class RecordFieldsUtil {
stringSerde.serialize(nestedName, fieldAbvs.getDataOutput());
// Value
valueAbvs.reset();
- ARecordType newType;
- if (fieldType == null) {
- newType = openType;
- } else {
- newType = (ARecordType) fieldType;
- }
+ ARecordType newType = (ARecordType) fieldType;
ARecordPointable recordP = getRecordPointable();
recordP.set(recordArg);
processRecord(recordP, newType, valueAbvs.getDataOutput(), level);
fieldRecordBuilder.addField(fieldAbvs, valueAbvs);
}
- public void processListValue(IValueReference listArg, IAType fieldType,
DataOutput out, int level)
+ public void processListValue(IValueReference listArg, IAType listType,
DataOutput out, int level)
throws IOException {
ArrayBackedValueStorage itemValue = getTempBuffer();
IARecordBuilder listRecordBuilder = getRecordBuilder();
- AListPointable list = getListPointable(fieldType.getTypeTag());
+ AListPointable list = getListPointable(listType.getTypeTag());
list.set(listArg);
OrderedListBuilder innerListBuilder = getOrderedListBuilder();
- innerListBuilder.reset(listType);
+ innerListBuilder.reset(RecordFieldsUtil.listType);
- listRecordBuilder.reset(null);
- AbstractCollectionType act = (AbstractCollectionType) fieldType;
+ listRecordBuilder.reset(openType);
+ AbstractCollectionType act = (AbstractCollectionType) listType;
int itemCount = list.getItemCount();
for (int l = 0; l < itemCount; l++) {
itemValue.reset();
@@ -269,11 +269,24 @@ public class RecordFieldsUtil {
byte tagId = list.getItemTag(act, l);
addFieldType(tagId, listRecordBuilder);
- if (tagId == ATypeTag.SERIALIZED_RECORD_TYPE_TAG) {
+ if (tagId == ATypeTag.SERIALIZED_RECORD_TYPE_TAG || tagId ==
ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG
+ || tagId == ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG) {
ArrayBackedValueStorage tmpAbvs = getTempBuffer();
tmpAbvs.reset();
list.getItemValue(act, l, tmpAbvs.getDataOutput());
- addNestedField(tmpAbvs, act.getItemType(), listRecordBuilder,
level + 1);
+ IAType itemType;
+
+ if (tagId == ATypeTag.SERIALIZED_RECORD_TYPE_TAG) {
+ itemType = act.getItemType().getTypeTag() != ATypeTag.ANY
? act.getItemType() : openType;
+ addRecordField(tmpAbvs, itemType, listRecordBuilder, level
+ 1);
+ } else if (tagId == ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG) {
+ itemType = act.getItemType().getTypeTag() != ATypeTag.ANY
? act.getItemType() : openOrderedListType;
+ addListField(tmpAbvs, itemType, listRecordBuilder, level +
1);
+ } else {
+ itemType =
+ act.getItemType().getTypeTag() != ATypeTag.ANY ?
act.getItemType() : openUnOrderedListType;
+ addListField(tmpAbvs, itemType, listRecordBuilder, level +
1);
+ }
}
listRecordBuilder.write(itemValue.getDataOutput(), true);
