>From Wail Alkowaileet <[email protected]>:
Wail Alkowaileet has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17902 )
Change subject: [ASTERIXDB-3298][RT] Fix IndexOutOfBoundException in COPY TO
......................................................................
[ASTERIXDB-3298][RT] Fix IndexOutOfBoundException in COPY TO
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
IndexOutOfBoundException is thrown if the write path is empty
Change-Id: Ic4d9b95220fdfd69465bd4cb4ecc4f2a7f2bf390
---
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.02.container.sqlpp
M
asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.04.update.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.01.container.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.05.ddl.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.06.query.sqlpp
M
asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/DynamicPathResolver.java
M
asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/StaticPathResolver.java
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.03.ddl.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/empty-path/empty-path.06.adm
M
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
11 files changed, 226 insertions(+), 8 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/02/17902/1
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
index 532da56..7892a17 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
@@ -481,10 +481,13 @@
String lastLine = lines[lines.length - 1];
String[] command = lastLine.trim().split(" ");
int length = command.length;
- if (length != 3) {
+ if (length == 1) {
+ createBucket(command[0]);
+ } else if (length == 3) {
+ dropRecreateBucket(command[0], command[1], command[2]);
+ } else {
throw new Exception("invalid create bucket format");
}
- dropRecreateBucket(command[0], command[1], command[2]);
break;
default:
super.executeTestFile(testCaseCtx, ctx, variableCtx,
statement, isDmlRecoveryTest, pb, cUnit,
@@ -493,10 +496,7 @@
}
}
- private static void dropRecreateBucket(String bucketName, String
definition, String files) {
- String definitionPath = definition + (definition.endsWith("/") ? "" :
"/");
- String[] fileSplits = files.split(",");
-
+ private static void createBucket(String bucketName) {
LOGGER.info("Dropping bucket " + bucketName);
try {
client.deleteBucket(DELETE_BUCKET_BUILDER.bucket(bucketName).build());
@@ -505,6 +505,12 @@
}
LOGGER.info("Creating bucket " + bucketName);
client.createBucket(CREATE_BUCKET_BUILDER.bucket(bucketName).build());
+ }
+
+ private static void dropRecreateBucket(String bucketName, String
definition, String files) {
+ String definitionPath = definition + (definition.endsWith("/") ? "" :
"/");
+ String[] fileSplits = files.split(",");
+ createBucket(bucketName);
LOGGER.info("Uploading to bucket " + bucketName + " definition " +
definitionPath);
fileNames.clear();
for (int i = 0; i < fileSplits.length; i++) {
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.01.container.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.01.container.sqlpp
new file mode 100644
index 0000000..664822e
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.01.container.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+// create empty bucket
+empty-bucket1
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.02.container.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.02.container.sqlpp
new file mode 100644
index 0000000..eb92908
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.02.container.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+// create empty bucket
+empty-bucket2
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.03.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.03.ddl.sqlpp
new file mode 100644
index 0000000..b68c38b
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.03.ddl.sqlpp
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+
+USE test;
+
+CREATE TYPE OpenType AS {
+};
+
+CREATE EXTERNAL DATASET Customer(OpenType) USING S3 (
+ ("accessKeyId"="dummyAccessKey"),
+ ("secretAccessKey"="dummySecretKey"),
+ ("region"="us-west-2"),
+ ("serviceEndpoint"="http://127.0.0.1:8001";),
+ ("container"="playground"),
+
("definition"="external-filter/car/{company:string}/customer/{customer_id:int}"),
+ ("embed-filter-values" = "false"),
+ ("format"="json")
+);
+
+
+
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.04.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.04.update.sqlpp
new file mode 100644
index 0000000..8ff9deb
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.04.update.sqlpp
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+USE test;
+
+COPY Customer c
+TO S3
+PATH ()
+WITH {
+ "accessKeyId":"dummyAccessKey",
+ "secretAccessKey":"dummySecretKey",
+ "region":"us-west-2",
+ "serviceEndpoint":"http://127.0.0.1:8001";,
+ "container":"empty-bucket1",
+ "format":"json"
+};
+
+COPY Customer c
+TO S3
+PATH ("")
+WITH {
+ "accessKeyId":"dummyAccessKey",
+ "secretAccessKey":"dummySecretKey",
+ "region":"us-west-2",
+ "serviceEndpoint":"http://127.0.0.1:8001";,
+ "container":"empty-bucket2",
+ "format":"json"
+};
+
+
+
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.05.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.05.ddl.sqlpp
new file mode 100644
index 0000000..54bad2f
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.05.ddl.sqlpp
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+CREATE EXTERNAL DATASET CustomerCopy1(OpenType) USING S3 (
+ ("accessKeyId"="dummyAccessKey"),
+ ("secretAccessKey"="dummySecretKey"),
+ ("region"="us-west-2"),
+ ("serviceEndpoint"="http://127.0.0.1:8001";),
+ ("container"="empty-bucket1"),
+ ("embed-filter-values" = "false"),
+ ("format"="json")
+);
+
+CREATE EXTERNAL DATASET CustomerCopy2(OpenType) USING S3 (
+ ("accessKeyId"="dummyAccessKey"),
+ ("secretAccessKey"="dummySecretKey"),
+ ("region"="us-west-2"),
+ ("serviceEndpoint"="http://127.0.0.1:8001";),
+ ("container"="empty-bucket2"),
+ ("embed-filter-values" = "false"),
+ ("format"="json")
+);
+
+
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.06.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.06.query.sqlpp
new file mode 100644
index 0000000..368265a
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.06.query.sqlpp
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SELECT (SELECT VALUE COUNT(*) FROM CustomerCopy1)[0] c1,
+ (SELECT VALUE COUNT(*) FROM CustomerCopy2)[0] c2
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/empty-path/empty-path.06.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/empty-path/empty-path.06.adm
new file mode 100644
index 0000000..37a540f
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/empty-path/empty-path.06.adm
@@ -0,0 +1 @@
+{ "c1": 81, "c2": 81 }
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index bd9e6ca..f0737d5 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -34,6 +34,11 @@
<output-dir compare="Text">query</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="copy-to">
+ <compilation-unit name="empty-path">
+ <output-dir compare="Text">empty-path</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="copy-to/negative">
<compilation-unit name="early-missing">
<output-dir compare="Text">early-missing</output-dir>
diff --git
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/DynamicPathResolver.java
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/DynamicPathResolver.java
index a1d2411..7105efa 100644
---
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/DynamicPathResolver.java
+++
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/DynamicPathResolver.java
@@ -57,7 +57,7 @@
return ExternalWriter.UNRESOLVABLE_PATH;
}
- if (dirStringBuilder.charAt(dirStringBuilder.length() - 1) !=
fileSeparator) {
+ if (dirStringBuilder.length() > 0 &&
dirStringBuilder.charAt(dirStringBuilder.length() - 1) != fileSeparator) {
dirStringBuilder.append(fileSeparator);
}
return dirStringBuilder.toString();
diff --git
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/StaticPathResolver.java
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/StaticPathResolver.java
index 4caf44f..52943ed 100644
---
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/StaticPathResolver.java
+++
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/StaticPathResolver.java
@@ -26,7 +26,7 @@
StaticPathResolver(String fileExtension, char fileSeparator, int
partition, String directoryPath) {
super(fileExtension, fileSeparator, partition);
- if (directoryPath.charAt(directoryPath.length() - 1) != fileSeparator)
{
+ if (!directoryPath.isEmpty() &&
directoryPath.charAt(directoryPath.length() - 1) != fileSeparator) {
this.directoryPath = directoryPath + fileSeparator;
} else {
this.directoryPath = directoryPath;
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17902
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: Ic4d9b95220fdfd69465bd4cb4ecc4f2a7f2bf390
Gerrit-Change-Number: 17902
Gerrit-PatchSet: 1
Gerrit-Owner: Wail Alkowaileet <[email protected]>
Gerrit-MessageType: newchange
