>From Wail Alkowaileet <[email protected]>: Wail Alkowaileet has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17902 )
Change subject: [ASTERIXDB-3298][RT] Fix IndexOutOfBoundException in COPY TO ...................................................................... [ASTERIXDB-3298][RT] Fix IndexOutOfBoundException in COPY TO - user model changes: no - storage format changes: no - interface changes: no Details: IndexOutOfBoundException is thrown if the write path is empty Change-Id: Ic4d9b95220fdfd69465bd4cb4ecc4f2a7f2bf390 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17902 Integration-Tests: Jenkins <[email protected]> Tested-by: Wail Alkowaileet <[email protected]> Reviewed-by: Wail Alkowaileet <[email protected]> --- A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.02.container.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.04.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.01.container.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.05.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.06.query.sqlpp M asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/DynamicPathResolver.java M asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/StaticPathResolver.java A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.03.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/empty-path/empty-path.06.adm M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java 11 files changed, 230 insertions(+), 8 deletions(-) Approvals: Wail Alkowaileet: Looks good to me, approved; Verified Jenkins: Verified diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java index 532da56..7892a17 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java @@ -481,10 +481,13 @@ String lastLine = lines[lines.length - 1]; String[] command = lastLine.trim().split(" "); int length = command.length; - if (length != 3) { + if (length == 1) { + createBucket(command[0]); + } else if (length == 3) { + dropRecreateBucket(command[0], command[1], command[2]); + } else { throw new Exception("invalid create bucket format"); } - dropRecreateBucket(command[0], command[1], command[2]); break; default: super.executeTestFile(testCaseCtx, ctx, variableCtx, statement, isDmlRecoveryTest, pb, cUnit, @@ -493,10 +496,7 @@ } } - private static void dropRecreateBucket(String bucketName, String definition, String files) { - String definitionPath = definition + (definition.endsWith("/") ? "" : "/"); - String[] fileSplits = files.split(","); - + private static void createBucket(String bucketName) { LOGGER.info("Dropping bucket " + bucketName); try { client.deleteBucket(DELETE_BUCKET_BUILDER.bucket(bucketName).build()); @@ -505,6 +505,12 @@ } LOGGER.info("Creating bucket " + bucketName); client.createBucket(CREATE_BUCKET_BUILDER.bucket(bucketName).build()); + } + + private static void dropRecreateBucket(String bucketName, String definition, String files) { + String definitionPath = definition + (definition.endsWith("/") ? "" : "/"); + String[] fileSplits = files.split(","); + createBucket(bucketName); LOGGER.info("Uploading to bucket " + bucketName + " definition " + definitionPath); fileNames.clear(); for (int i = 0; i < fileSplits.length; i++) { diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.01.container.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.01.container.sqlpp new file mode 100644 index 0000000..664822e --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.01.container.sqlpp @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +// create empty bucket +empty-bucket1 \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.02.container.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.02.container.sqlpp new file mode 100644 index 0000000..eb92908 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.02.container.sqlpp @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +// create empty bucket +empty-bucket2 \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.03.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.03.ddl.sqlpp new file mode 100644 index 0000000..b68c38b --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.03.ddl.sqlpp @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +DROP DATAVERSE test IF EXISTS; +CREATE DATAVERSE test; + +USE test; + +CREATE TYPE OpenType AS { +}; + +CREATE EXTERNAL DATASET Customer(OpenType) USING S3 ( + ("accessKeyId"="dummyAccessKey"), + ("secretAccessKey"="dummySecretKey"), + ("region"="us-west-2"), + ("serviceEndpoint"="http://127.0.0.1:8001";), + ("container"="playground"), + ("definition"="external-filter/car/{company:string}/customer/{customer_id:int}"), + ("embed-filter-values" = "false"), + ("format"="json") +); + + + diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.04.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.04.update.sqlpp new file mode 100644 index 0000000..8ff9deb --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.04.update.sqlpp @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +USE test; + +COPY Customer c +TO S3 +PATH () +WITH { + "accessKeyId":"dummyAccessKey", + "secretAccessKey":"dummySecretKey", + "region":"us-west-2", + "serviceEndpoint":"http://127.0.0.1:8001";, + "container":"empty-bucket1", + "format":"json" +}; + +COPY Customer c +TO S3 +PATH ("") +WITH { + "accessKeyId":"dummyAccessKey", + "secretAccessKey":"dummySecretKey", + "region":"us-west-2", + "serviceEndpoint":"http://127.0.0.1:8001";, + "container":"empty-bucket2", + "format":"json" +}; + + + diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.05.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.05.ddl.sqlpp new file mode 100644 index 0000000..54bad2f --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.05.ddl.sqlpp @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; + +CREATE EXTERNAL DATASET CustomerCopy1(OpenType) USING S3 ( + ("accessKeyId"="dummyAccessKey"), + ("secretAccessKey"="dummySecretKey"), + ("region"="us-west-2"), + ("serviceEndpoint"="http://127.0.0.1:8001";), + ("container"="empty-bucket1"), + ("embed-filter-values" = "false"), + ("format"="json") +); + +CREATE EXTERNAL DATASET CustomerCopy2(OpenType) USING S3 ( + ("accessKeyId"="dummyAccessKey"), + ("secretAccessKey"="dummySecretKey"), + ("region"="us-west-2"), + ("serviceEndpoint"="http://127.0.0.1:8001";), + ("container"="empty-bucket2"), + ("embed-filter-values" = "false"), + ("format"="json") +); + + diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.06.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.06.query.sqlpp new file mode 100644 index 0000000..368265a --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.06.query.sqlpp @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; + +SELECT (SELECT VALUE COUNT(*) FROM CustomerCopy1)[0] c1, + (SELECT VALUE COUNT(*) FROM CustomerCopy2)[0] c2 \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/empty-path/empty-path.06.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/empty-path/empty-path.06.adm new file mode 100644 index 0000000..37a540f --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/empty-path/empty-path.06.adm @@ -0,0 +1 @@ +{ "c1": 81, "c2": 81 } diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml index 54ccd30..421aacd 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml @@ -39,6 +39,11 @@ <output-dir compare="Text">default-namespace</output-dir> </compilation-unit> </test-case> + <test-case FilePath="copy-to"> + <compilation-unit name="empty-path"> + <output-dir compare="Text">empty-path</output-dir> + </compilation-unit> + </test-case> <test-case FilePath="copy-to/negative"> <compilation-unit name="early-missing"> <output-dir compare="Text">early-missing</output-dir> diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/DynamicPathResolver.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/DynamicPathResolver.java index a1d2411..7105efa 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/DynamicPathResolver.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/DynamicPathResolver.java @@ -57,7 +57,7 @@ return ExternalWriter.UNRESOLVABLE_PATH; } - if (dirStringBuilder.charAt(dirStringBuilder.length() - 1) != fileSeparator) { + if (dirStringBuilder.length() > 0 && dirStringBuilder.charAt(dirStringBuilder.length() - 1) != fileSeparator) { dirStringBuilder.append(fileSeparator); } return dirStringBuilder.toString(); diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/StaticPathResolver.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/StaticPathResolver.java index 4caf44f..52943ed 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/StaticPathResolver.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/StaticPathResolver.java @@ -26,7 +26,7 @@ StaticPathResolver(String fileExtension, char fileSeparator, int partition, String directoryPath) { super(fileExtension, fileSeparator, partition); - if (directoryPath.charAt(directoryPath.length() - 1) != fileSeparator) { + if (!directoryPath.isEmpty() && directoryPath.charAt(directoryPath.length() - 1) != fileSeparator) { this.directoryPath = directoryPath + fileSeparator; } else { this.directoryPath = directoryPath; -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17902 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: Ic4d9b95220fdfd69465bd4cb4ecc4f2a7f2bf390 Gerrit-Change-Number: 17902 Gerrit-PatchSet: 3 Gerrit-Owner: Wail Alkowaileet <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Hussain Towaileb <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Wail Alkowaileet <[email protected]> Gerrit-MessageType: merged
