This is an automated email from the ASF dual-hosted git repository.
ggalvizo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new d80b5de [ASTERIXDB-2932][IDX] Fixing open type building for array
indexes.
d80b5de is described below
commit d80b5de2615d74106d17f72bf93a4cdb0439b8a4
Author: ggalvizo <[email protected]>
AuthorDate: Mon Jul 19 09:41:35 2021 -1000
[ASTERIXDB-2932][IDX] Fixing open type building for array indexes.
- user mode changes: no
- storage format changes: no
- interface changes: no
Fix to allow for more complex nesting structures for array indexes
(partially typed indexes + records that hold subrecords that hold
arrays). This also fixes the open type names to be more consistent
with the closed types.
Change-Id: If6cf7d13081a6f58457ceefed5c090208fedf8a0
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12403
Tested-by: Jenkins <[email protected]>
Integration-Tests: Jenkins <[email protected]>
Reviewed-by: Glenn Galvizo <[email protected]>
Reviewed-by: Dmitry Lychagin <[email protected]>
---
.../complex-structures-01.1.ddl.sqlpp | 79 ++++++++++++++++++++++
.../complex-structures-01.2.query.sqlpp | 24 +++++++
.../complex-structures/complex-structures-01.1.adm | 8 +++
.../test/resources/runtimets/testsuite_sqlpp.xml | 5 ++
.../apache/asterix/metadata/utils/TypeUtil.java | 51 ++++++++------
5 files changed, 147 insertions(+), 20 deletions(-)
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/metadata/open/complex-structures/complex-structures-01.1.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/metadata/open/complex-structures/complex-structures-01.1.ddl.sqlpp
new file mode 100644
index 0000000..c0a4d1d
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/metadata/open/complex-structures/complex-structures-01.1.ddl.sqlpp
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+DROP DATAVERSE TestDataverse IF EXISTS;
+CREATE DATAVERSE TestDataverse;
+USE TestDataverse;
+CREATE TYPE TestType AS {
+ _id: uuid,
+ closed_field_1: {
+ closed_field_2: [{
+ closed_field_3: {
+ closed_field_4: [{
+ closed_field_5: bigint
+ }]
+ }
+ }]
+ }
+};
+CREATE DATASET TestDataset (TestType)
+PRIMARY KEY _id AUTOGENERATED;
+
+
+-- Fully open index 1. Unnest flags: [0, 1, 0]
+CREATE INDEX testIndex1
+ON TestDataset ( UNNEST open_field_1.open_field_2
+ SELECT open_field_3a : bigint );
+
+-- Fully open index 2. Unnest flags: [0, 1, 0, 0]
+CREATE INDEX testIndex2
+ON TestDataset ( UNNEST open_field_1.open_field_2
+ SELECT open_field_3b.open_field_4 : bigint );
+
+-- Fully open index 3. Unnest flags: [0, 1, 0, 1]
+CREATE INDEX testIndex3
+ON TestDataset ( UNNEST open_field_1.open_field_2
+ UNNEST open_field_3c.open_field_4a : bigint );
+
+-- Fully open index 4. Unnest flags: [0, 1, 0, 1, 0]
+CREATE INDEX testIndex4
+ON TestDataset ( UNNEST open_field_1.open_field_2
+ UNNEST open_field_3c.open_field_4b
+ SELECT open_field_5 : bigint );
+
+
+-- Partially open index 1. [0, 1, 0]
+CREATE INDEX testIndex1c
+ON TestDataset ( UNNEST closed_field_1.open_field_2
+ SELECT open_field_3a : bigint );
+
+-- Partially open index 2. Unnest flags: [0, 1, 0, 0]
+CREATE INDEX testIndex2c
+ON TestDataset ( UNNEST closed_field_1.closed_field_2
+ SELECT open_field_3b.open_field_4 : bigint );
+
+-- Partially open index 3. Unnest flags: [0, 1, 0, 1]
+CREATE INDEX testIndex3c
+ON TestDataset ( UNNEST closed_field_1.closed_field_2
+ UNNEST closed_field_3.open_field_4a : bigint );
+
+-- Partially open index 4. Unnest flags: [0, 1, 0, 1, 0]
+CREATE INDEX testIndex4c
+ON TestDataset ( UNNEST closed_field_1.closed_field_2
+ UNNEST closed_field_3.closed_field_4
+ SELECT open_field_5 : bigint );
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/metadata/open/complex-structures/complex-structures-01.2.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/metadata/open/complex-structures/complex-structures-01.2.query.sqlpp
new file mode 100644
index 0000000..1b854c3
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/metadata/open/complex-structures/complex-structures-01.2.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+FROM Metadata.`Index` D
+WHERE D.IndexName LIKE "testIndex%" AND
+ D.DataverseName = "TestDataverse"
+SELECT D.SearchKey, D.SearchKeyElements, D.SearchKeyType
+ORDER BY D.IndexName ASC;
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/metadata/open/complex-structures/complex-structures-01.1.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/metadata/open/complex-structures/complex-structures-01.1.adm
new file mode 100644
index 0000000..7618b68
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/metadata/open/complex-structures/complex-structures-01.1.adm
@@ -0,0 +1,8 @@
+{ "SearchKey": [ ], "SearchKeyElements": [ { "UnnestList": [ [
"open_field_1", "open_field_2" ] ], "ProjectList": [ [ "open_field_3a" ] ] } ],
"SearchKeyType": [ [ "int64" ] ] }
+{ "SearchKey": [ ], "SearchKeyElements": [ { "UnnestList": [ [
"closed_field_1", "open_field_2" ] ], "ProjectList": [ [ "open_field_3a" ] ] }
], "SearchKeyType": [ [ "int64" ] ] }
+{ "SearchKey": [ ], "SearchKeyElements": [ { "UnnestList": [ [
"open_field_1", "open_field_2" ] ], "ProjectList": [ [ "open_field_3b",
"open_field_4" ] ] } ], "SearchKeyType": [ [ "int64" ] ] }
+{ "SearchKey": [ ], "SearchKeyElements": [ { "UnnestList": [ [
"closed_field_1", "closed_field_2" ] ], "ProjectList": [ [ "open_field_3b",
"open_field_4" ] ] } ], "SearchKeyType": [ [ "int64" ] ] }
+{ "SearchKey": [ ], "SearchKeyElements": [ { "UnnestList": [ [
"open_field_1", "open_field_2" ], [ "open_field_3c", "open_field_4a" ] ] } ],
"SearchKeyType": [ [ "int64" ] ] }
+{ "SearchKey": [ ], "SearchKeyElements": [ { "UnnestList": [ [
"closed_field_1", "closed_field_2" ], [ "closed_field_3", "open_field_4a" ] ] }
], "SearchKeyType": [ [ "int64" ] ] }
+{ "SearchKey": [ ], "SearchKeyElements": [ { "UnnestList": [ [
"open_field_1", "open_field_2" ], [ "open_field_3c", "open_field_4b" ] ],
"ProjectList": [ [ "open_field_5" ] ] } ], "SearchKeyType": [ [ "int64" ] ] }
+{ "SearchKey": [ ], "SearchKeyElements": [ { "UnnestList": [ [
"closed_field_1", "closed_field_2" ], [ "closed_field_3", "closed_field_4" ] ],
"ProjectList": [ [ "open_field_5" ] ] } ], "SearchKeyType": [ [ "int64" ] ] }
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 74dc47b..03be890 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -8066,6 +8066,11 @@
<output-dir compare="Text">use-case-4</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="array-index/metadata/open">
+ <compilation-unit name="complex-structures">
+ <output-dir compare="Text">complex-structures</output-dir>
+ </compilation-unit>
+ </test-case>
<!-- <test-case FilePath="array-index/metadata/closed">-->
<!-- <compilation-unit name="with-composite-sk">-->
<!-- <output-dir
compare="Text">with-composite-sk</output-dir>-->
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TypeUtil.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TypeUtil.java
index b1d610f..8a9236e 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TypeUtil.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TypeUtil.java
@@ -131,23 +131,26 @@ public class TypeUtil {
}
private IAType buildNewForOpenType() {
- boolean isTypeWithUnnest = keyUnnestFlags.subList(indexOfOpenPart
+ 1, keyUnnestFlags.size()).stream()
- .filter(i -> i).findFirst().orElse(false);
- IAType resultant = nestArrayType(keyFieldType, isTypeWithUnnest);
-
- // Build the type (list or record) that holds the type (list or
record) above.
- resultant = nestArrayType(
- new ARecordType(keyFieldNames.get(keyFieldNames.size() -
2),
- new String[] {
keyFieldNames.get(keyFieldNames.size() - 1) },
- new IAType[] {
AUnionType.createUnknownableType(resultant) }, true),
- keyUnnestFlags.get(indexOfOpenPart));
-
- // Create open part of the nested field.
- for (int i = keyFieldNames.size() - 3; i > (indexOfOpenPart - 1);
i--) {
- resultant = nestArrayType(
- new ARecordType(keyFieldNames.get(i), new String[] {
keyFieldNames.get(i + 1) },
- new IAType[] {
AUnionType.createUnknownableType(resultant) }, true),
- keyUnnestFlags.get(i));
+ // Walk backwards through our flags and construct the desired type.
+ List<Boolean> unnestFlagsForOpenType =
keyUnnestFlags.subList(indexOfOpenPart, keyUnnestFlags.size());
+ List<String> fieldNamesForOpenType =
keyFieldNames.subList(indexOfOpenPart, keyFieldNames.size());
+ IAType resultant = keyFieldType;
+ for (int i = unnestFlagsForOpenType.size() - 1; i >= 0; i--) {
+ // Construct the type name.
+ StringBuilder recordTypeNameBuilder = new StringBuilder();
+ recordTypeNameBuilder.append(baseRecordType.getTypeName());
+ for (int j = 0; j < i + indexOfOpenPart; j++) {
+
recordTypeNameBuilder.append("_").append(keyFieldNames.get(j));
+ if (keyUnnestFlags.get(j)) {
+ recordTypeNameBuilder.append("_Item");
+ }
+ }
+
+ // Construct the type itself and account for any array steps.
+ resultant = nestArrayType(resultant,
unnestFlagsForOpenType.get(i));
+ resultant =
+ new ARecordType(recordTypeNameBuilder.toString(), new
String[] { fieldNamesForOpenType.get(i) },
+ new IAType[] {
AUnionType.createUnknownableType(resultant) }, true);
}
// Now update the parent to include this optional field,
accounting for intermediate arrays.
@@ -155,9 +158,9 @@ public class TypeUtil {
ARecordType parentRecord =
(ARecordType)
unnestArrayType(TypeComputeUtils.getActualType(gapTriple.first),
gapTriple.third);
IAType[] parentFieldTypes =
ArrayUtils.addAll(parentRecord.getFieldTypes().clone(),
- AUnionType.createUnknownableType(resultant));
+ ((ARecordType) resultant).getFieldTypes().clone());
resultant = new ARecordType(bridgeNameFoundFromOpenTypeBuild,
- ArrayUtils.addAll(parentRecord.getFieldNames(),
resultant.getTypeName()), parentFieldTypes, true);
+ ArrayUtils.addAll(parentRecord.getFieldNames(),
gapTriple.second), parentFieldTypes, true);
resultant = keepUnknown(gapTriple.first, nestArrayType(resultant,
gapTriple.third));
return resultant;
@@ -227,7 +230,15 @@ public class TypeUtil {
}
private static IAType nestArrayType(IAType originalType, boolean
isWithinArray) {
- return (isWithinArray) ? new AOrderedListType(originalType,
originalType.getTypeName()) : originalType;
+ if (isWithinArray) {
+ String newTypeName =
originalType.getTypeName().endsWith("_Item")
+ ? originalType.getTypeName().substring(0,
originalType.getTypeName().length() - 5)
+ : originalType.getTypeName();
+ return new AOrderedListType(originalType, newTypeName +
"_Array");
+
+ } else {
+ return originalType;
+ }
}
private static IAType unnestArrayType(IAType originalType, boolean
isWithinArray) {
