This is an automated email from the ASF dual-hosted git repository.
ggalvizo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 2ee6d37 [ASTERIXDB-2959][IDX] Fix initial composite atomic-array
index ingestion on creation
2ee6d37 is described below
commit 2ee6d37456c6149cd088d236b12bcb6701d2004f
Author: ggalvizo <[email protected]>
AuthorDate: Fri Sep 10 12:58:38 2021 -0700
[ASTERIXDB-2959][IDX] Fix initial composite atomic-array index ingestion on
creation
- user model changes: no
- storage format changes: no
- interface changes: no
Composite atomic-array indexes were not ingesting properly, due to the
wrong source column being UNNESTed in the loading job. This small change
addresses this.
Change-Id: I86af26a7649e0631a0555e1f98fb5fea8e63528e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/13163
Tested-by: Jenkins <[email protected]>
Integration-Tests: Jenkins <[email protected]>
Reviewed-by: Dmitry Lychagin <[email protected]>
---
.../composite-atomic/composite-atomic.1.ddl.sqlpp | 25 ++++++++++++++++++++
.../composite-atomic.2.update.sqlpp | 27 ++++++++++++++++++++++
.../composite-atomic/composite-atomic.3.ddl.sqlpp | 26 +++++++++++++++++++++
.../composite-atomic.4.query.sqlpp | 25 ++++++++++++++++++++
.../open/composite-atomic/composite-atomic.1.adm | 1 +
.../test/resources/runtimets/testsuite_sqlpp.xml | 9 ++++++--
.../asterix/metadata/utils/ArrayIndexUtil.java | 2 +-
7 files changed, 112 insertions(+), 3 deletions(-)
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.1.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.1.ddl.sqlpp
new file mode 100644
index 0000000..6312915
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.1.ddl.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+DROP DATAVERSE TestDataverse IF EXISTS;
+CREATE DATAVERSE TestDataverse;
+USE TestDataverse;
+
+CREATE TYPE GenericType AS { _id: bigint };
+CREATE DATASET TestDataset (GenericType)
+PRIMARY KEY _id;
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.2.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.2.update.sqlpp
new file mode 100644
index 0000000..e901b37
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.2.update.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+USE TestDataverse;
+
+INSERT INTO TestDataset [
+ { "_id": 100, "f": 1, "a": {"b": [{"c": {"d": 1}}]} },
+ { "_id": 101, "f": "a", "a": null },
+ { "_id": 102, "f": "a", "a": {"b": null} },
+ { "_id": 103, "f": "a", "a": {"b": [{"c": null}]} },
+ { "_id": 104, "f": "a", "a": {"b": [{"c": {"d": null}}]} }
+];
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.3.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.3.ddl.sqlpp
new file mode 100644
index 0000000..8bcf2c5
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.3.ddl.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+USE TestDataverse;
+
+CREATE INDEX TestIdx
+ON TestDataset (
+ f : bigint,
+ UNNEST a.b
+ SELECT c.d : bigint
+);
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.4.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.4.query.sqlpp
new file mode 100644
index 0000000..a5de0da
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.4.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+SET `compiler.arrayindex` "true";
+USE TestDataverse;
+
+FROM TestDataset D
+UNNEST D.a.b AB
+WHERE D.f = 1 AND AB.c.d = 1
+SELECT D._id;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.1.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.1.adm
new file mode 100644
index 0000000..32c3a65
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.1.adm
@@ -0,0 +1 @@
+{ "_id": 100 }
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 6879938..4220f99 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -8217,8 +8217,13 @@
</compilation-unit>
</test-case>
<test-case FilePath="array-index/bulk-loading/on-index-creation/closed">
- <compilation-unit name="with-3-level-record-path">
- <output-dir compare="Text">with-3-level-record-path</output-dir>
+ <compilation-unit name="with-3-level-record-path">
+ <output-dir compare="Text">with-3-level-record-path</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/on-index-creation/open">
+ <compilation-unit name="composite-atomic">
+ <output-dir compare="Text">composite-atomic</output-dir>
</compilation-unit>
</test-case>
</test-group>
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java
index ab181cb..7282c7d 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java
@@ -245,6 +245,7 @@ public class ArrayIndexUtil {
ArrayPath arrayPath = new ArrayPath(flattenedFieldName,
unnestFlags).invoke();
List<List<String>> fieldNamesPerArray = arrayPath.fieldNamesPerArray;
List<Boolean> unnestFlagsPerArray = arrayPath.unnestFlagsPerArray;
+ boolean requiresOnlyOneUnnest = unnestFlags.stream().filter(f ->
f).count() == 1;
// If we are given no base record type, then we do not need to keep
track of the record type. We are solely
// using this walk for its flags.
@@ -290,7 +291,6 @@ public class ArrayIndexUtil {
}
if (i == fieldNamesPerArray.size() - 1) {
- boolean requiresOnlyOneUnnest = fieldNamesPerArray.size() == 1;
boolean isNonArrayStep = !unnestFlagsPerArray.get(i);
commandExecutor.executeActionOnFinalArrayStep(startingStepRecordType,
fieldNamesPerArray.get(i),
isNonArrayStep, requiresOnlyOneUnnest);
