This is an automated email from the ASF dual-hosted git repository.
wyk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 80c24638e2 [ASTERIXDB-3298][RT] Fix IndexOutOfBoundException in COPY TO
80c24638e2 is described below
commit 80c24638e247064a0a5e6c4036874e12f682730d
Author: Wail Alkowaileet <[email protected]>
AuthorDate: Fri Nov 3 08:46:49 2023 -0700
[ASTERIXDB-3298][RT] Fix IndexOutOfBoundException in COPY TO
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
IndexOutOfBoundException is thrown if the write path is empty
Change-Id: Ic4d9b95220fdfd69465bd4cb4ecc4f2a7f2bf390
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17902
Integration-Tests: Jenkins <[email protected]>
Tested-by: Wail Alkowaileet <[email protected]>
Reviewed-by: Wail Alkowaileet <[email protected]>
---
.../aws/AwsS3ExternalDatasetTest.java | 18 ++++++----
.../empty-path/empty-path.01.container.sqlpp} | 24 ++-----------
.../empty-path/empty-path.02.container.sqlpp} | 24 ++-----------
.../copy-to/empty-path/empty-path.03.ddl.sqlpp} | 33 +++++++++---------
.../copy-to/empty-path/empty-path.04.update.sqlpp} | 40 +++++++++++++---------
.../copy-to/empty-path/empty-path.05.ddl.sqlpp} | 36 ++++++++++---------
.../copy-to/empty-path/empty-path.06.query.sqlpp} | 23 ++-----------
.../results/copy-to/empty-path/empty-path.06.adm | 1 +
.../runtimets/testsuite_external_dataset_s3.xml | 5 +++
.../runtime/writer/DynamicPathResolver.java | 2 +-
.../asterix/runtime/writer/StaticPathResolver.java | 2 +-
11 files changed, 85 insertions(+), 123 deletions(-)
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
index 532da56cf4..7892a177de 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
@@ -481,10 +481,13 @@ public class AwsS3ExternalDatasetTest {
String lastLine = lines[lines.length - 1];
String[] command = lastLine.trim().split(" ");
int length = command.length;
- if (length != 3) {
+ if (length == 1) {
+ createBucket(command[0]);
+ } else if (length == 3) {
+ dropRecreateBucket(command[0], command[1], command[2]);
+ } else {
throw new Exception("invalid create bucket format");
}
- dropRecreateBucket(command[0], command[1], command[2]);
break;
default:
super.executeTestFile(testCaseCtx, ctx, variableCtx,
statement, isDmlRecoveryTest, pb, cUnit,
@@ -493,10 +496,7 @@ public class AwsS3ExternalDatasetTest {
}
}
- private static void dropRecreateBucket(String bucketName, String
definition, String files) {
- String definitionPath = definition + (definition.endsWith("/") ? "" :
"/");
- String[] fileSplits = files.split(",");
-
+ private static void createBucket(String bucketName) {
LOGGER.info("Dropping bucket " + bucketName);
try {
client.deleteBucket(DELETE_BUCKET_BUILDER.bucket(bucketName).build());
@@ -505,6 +505,12 @@ public class AwsS3ExternalDatasetTest {
}
LOGGER.info("Creating bucket " + bucketName);
client.createBucket(CREATE_BUCKET_BUILDER.bucket(bucketName).build());
+ }
+
+ private static void dropRecreateBucket(String bucketName, String
definition, String files) {
+ String definitionPath = definition + (definition.endsWith("/") ? "" :
"/");
+ String[] fileSplits = files.split(",");
+ createBucket(bucketName);
LOGGER.info("Uploading to bucket " + bucketName + " definition " +
definitionPath);
fileNames.clear();
for (int i = 0; i < fileSplits.length; i++) {
diff --git
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/StaticPathResolver.java
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.01.container.sqlpp
similarity index 52%
copy from
asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/StaticPathResolver.java
copy to
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.01.container.sqlpp
index 4caf44f816..664822e9b3 100644
---
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/StaticPathResolver.java
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.01.container.sqlpp
@@ -16,25 +16,5 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.runtime.writer;
-
-import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-
-final class StaticPathResolver extends AbstractPathResolver {
- private final String directoryPath;
-
- StaticPathResolver(String fileExtension, char fileSeparator, int
partition, String directoryPath) {
- super(fileExtension, fileSeparator, partition);
-
- if (directoryPath.charAt(directoryPath.length() - 1) != fileSeparator)
{
- this.directoryPath = directoryPath + fileSeparator;
- } else {
- this.directoryPath = directoryPath;
- }
- }
-
- @Override
- public String getPartitionDirectory(IFrameTupleReference tuple) {
- return directoryPath;
- }
-}
+// create empty bucket
+empty-bucket1
\ No newline at end of file
diff --git
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/StaticPathResolver.java
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.02.container.sqlpp
similarity index 52%
copy from
asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/StaticPathResolver.java
copy to
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.02.container.sqlpp
index 4caf44f816..eb929081f7 100644
---
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/StaticPathResolver.java
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.02.container.sqlpp
@@ -16,25 +16,5 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.runtime.writer;
-
-import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-
-final class StaticPathResolver extends AbstractPathResolver {
- private final String directoryPath;
-
- StaticPathResolver(String fileExtension, char fileSeparator, int
partition, String directoryPath) {
- super(fileExtension, fileSeparator, partition);
-
- if (directoryPath.charAt(directoryPath.length() - 1) != fileSeparator)
{
- this.directoryPath = directoryPath + fileSeparator;
- } else {
- this.directoryPath = directoryPath;
- }
- }
-
- @Override
- public String getPartitionDirectory(IFrameTupleReference tuple) {
- return directoryPath;
- }
-}
+// create empty bucket
+empty-bucket2
\ No newline at end of file
diff --git
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/StaticPathResolver.java
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.03.ddl.sqlpp
similarity index 52%
copy from
asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/StaticPathResolver.java
copy to
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.03.ddl.sqlpp
index 4caf44f816..b68c38b195 100644
---
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/StaticPathResolver.java
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.03.ddl.sqlpp
@@ -16,25 +16,24 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.runtime.writer;
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
-import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+USE test;
-final class StaticPathResolver extends AbstractPathResolver {
- private final String directoryPath;
+CREATE TYPE OpenType AS {
+};
+
+CREATE EXTERNAL DATASET Customer(OpenType) USING S3 (
+ ("accessKeyId"="dummyAccessKey"),
+ ("secretAccessKey"="dummySecretKey"),
+ ("region"="us-west-2"),
+ ("serviceEndpoint"="http://127.0.0.1:8001";),
+ ("container"="playground"),
+
("definition"="external-filter/car/{company:string}/customer/{customer_id:int}"),
+ ("embed-filter-values" = "false"),
+ ("format"="json")
+);
- StaticPathResolver(String fileExtension, char fileSeparator, int
partition, String directoryPath) {
- super(fileExtension, fileSeparator, partition);
- if (directoryPath.charAt(directoryPath.length() - 1) != fileSeparator)
{
- this.directoryPath = directoryPath + fileSeparator;
- } else {
- this.directoryPath = directoryPath;
- }
- }
- @Override
- public String getPartitionDirectory(IFrameTupleReference tuple) {
- return directoryPath;
- }
-}
diff --git
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/StaticPathResolver.java
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.04.update.sqlpp
similarity index 52%
copy from
asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/StaticPathResolver.java
copy to
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.04.update.sqlpp
index 4caf44f816..8ff9deb3cb 100644
---
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/StaticPathResolver.java
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.04.update.sqlpp
@@ -16,25 +16,31 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.runtime.writer;
+USE test;
-import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+COPY Customer c
+TO S3
+PATH ()
+WITH {
+ "accessKeyId":"dummyAccessKey",
+ "secretAccessKey":"dummySecretKey",
+ "region":"us-west-2",
+ "serviceEndpoint":"http://127.0.0.1:8001";,
+ "container":"empty-bucket1",
+ "format":"json"
+};
-final class StaticPathResolver extends AbstractPathResolver {
- private final String directoryPath;
+COPY Customer c
+TO S3
+PATH ("")
+WITH {
+ "accessKeyId":"dummyAccessKey",
+ "secretAccessKey":"dummySecretKey",
+ "region":"us-west-2",
+ "serviceEndpoint":"http://127.0.0.1:8001";,
+ "container":"empty-bucket2",
+ "format":"json"
+};
- StaticPathResolver(String fileExtension, char fileSeparator, int
partition, String directoryPath) {
- super(fileExtension, fileSeparator, partition);
- if (directoryPath.charAt(directoryPath.length() - 1) != fileSeparator)
{
- this.directoryPath = directoryPath + fileSeparator;
- } else {
- this.directoryPath = directoryPath;
- }
- }
- @Override
- public String getPartitionDirectory(IFrameTupleReference tuple) {
- return directoryPath;
- }
-}
diff --git
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/StaticPathResolver.java
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.05.ddl.sqlpp
similarity index 52%
copy from
asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/StaticPathResolver.java
copy to
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.05.ddl.sqlpp
index 4caf44f816..54bad2f62a 100644
---
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/StaticPathResolver.java
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.05.ddl.sqlpp
@@ -16,25 +16,27 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.runtime.writer;
-import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+USE test;
-final class StaticPathResolver extends AbstractPathResolver {
- private final String directoryPath;
+CREATE EXTERNAL DATASET CustomerCopy1(OpenType) USING S3 (
+ ("accessKeyId"="dummyAccessKey"),
+ ("secretAccessKey"="dummySecretKey"),
+ ("region"="us-west-2"),
+ ("serviceEndpoint"="http://127.0.0.1:8001";),
+ ("container"="empty-bucket1"),
+ ("embed-filter-values" = "false"),
+ ("format"="json")
+);
- StaticPathResolver(String fileExtension, char fileSeparator, int
partition, String directoryPath) {
- super(fileExtension, fileSeparator, partition);
+CREATE EXTERNAL DATASET CustomerCopy2(OpenType) USING S3 (
+ ("accessKeyId"="dummyAccessKey"),
+ ("secretAccessKey"="dummySecretKey"),
+ ("region"="us-west-2"),
+ ("serviceEndpoint"="http://127.0.0.1:8001";),
+ ("container"="empty-bucket2"),
+ ("embed-filter-values" = "false"),
+ ("format"="json")
+);
- if (directoryPath.charAt(directoryPath.length() - 1) != fileSeparator)
{
- this.directoryPath = directoryPath + fileSeparator;
- } else {
- this.directoryPath = directoryPath;
- }
- }
- @Override
- public String getPartitionDirectory(IFrameTupleReference tuple) {
- return directoryPath;
- }
-}
diff --git
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/StaticPathResolver.java
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.06.query.sqlpp
similarity index 52%
copy from
asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/StaticPathResolver.java
copy to
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.06.query.sqlpp
index 4caf44f816..368265abdc 100644
---
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/StaticPathResolver.java
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.06.query.sqlpp
@@ -16,25 +16,8 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.runtime.writer;
-import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+USE test;
-final class StaticPathResolver extends AbstractPathResolver {
- private final String directoryPath;
-
- StaticPathResolver(String fileExtension, char fileSeparator, int
partition, String directoryPath) {
- super(fileExtension, fileSeparator, partition);
-
- if (directoryPath.charAt(directoryPath.length() - 1) != fileSeparator)
{
- this.directoryPath = directoryPath + fileSeparator;
- } else {
- this.directoryPath = directoryPath;
- }
- }
-
- @Override
- public String getPartitionDirectory(IFrameTupleReference tuple) {
- return directoryPath;
- }
-}
+SELECT (SELECT VALUE COUNT(*) FROM CustomerCopy1)[0] c1,
+ (SELECT VALUE COUNT(*) FROM CustomerCopy2)[0] c2
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/empty-path/empty-path.06.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/empty-path/empty-path.06.adm
new file mode 100644
index 0000000000..37a540f916
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/empty-path/empty-path.06.adm
@@ -0,0 +1 @@
+{ "c1": 81, "c2": 81 }
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index 54ccd30548..421aacddeb 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -39,6 +39,11 @@
<output-dir compare="Text">default-namespace</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="copy-to">
+ <compilation-unit name="empty-path">
+ <output-dir compare="Text">empty-path</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="copy-to/negative">
<compilation-unit name="early-missing">
<output-dir compare="Text">early-missing</output-dir>
diff --git
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/DynamicPathResolver.java
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/DynamicPathResolver.java
index a1d241124f..7105efacdd 100644
---
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/DynamicPathResolver.java
+++
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/DynamicPathResolver.java
@@ -57,7 +57,7 @@ final class DynamicPathResolver extends AbstractPathResolver {
return ExternalWriter.UNRESOLVABLE_PATH;
}
- if (dirStringBuilder.charAt(dirStringBuilder.length() - 1) !=
fileSeparator) {
+ if (dirStringBuilder.length() > 0 &&
dirStringBuilder.charAt(dirStringBuilder.length() - 1) != fileSeparator) {
dirStringBuilder.append(fileSeparator);
}
return dirStringBuilder.toString();
diff --git
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/StaticPathResolver.java
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/StaticPathResolver.java
index 4caf44f816..52943edbbc 100644
---
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/StaticPathResolver.java
+++
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/StaticPathResolver.java
@@ -26,7 +26,7 @@ final class StaticPathResolver extends AbstractPathResolver {
StaticPathResolver(String fileExtension, char fileSeparator, int
partition, String directoryPath) {
super(fileExtension, fileSeparator, partition);
- if (directoryPath.charAt(directoryPath.length() - 1) != fileSeparator)
{
+ if (!directoryPath.isEmpty() &&
directoryPath.charAt(directoryPath.length() - 1) != fileSeparator) {
this.directoryPath = directoryPath + fileSeparator;
} else {
this.directoryPath = directoryPath;
