This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 831e86c780 [ASTERIXDB-3566][EXT]: Add support to COPY TO for Azure
831e86c780 is described below
commit 831e86c780739dcb46b1e95959e397d2cd13fade
Author: Hussain Towaileb <[email protected]>
AuthorDate: Sat Sep 14 04:18:40 2024 +0300
[ASTERIXDB-3566][EXT]: Add support to COPY TO for Azure
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Add support to COPY TO for Azure.
- Enable all COPY TO test cases for Azure.
Ext-ref: MB-63080
Change-Id: I613d45f3f97bfdc2e7c2e7e4733d8cfce31ec359
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19449
Integration-Tests: Jenkins <[email protected]>
Tested-by: Jenkins <[email protected]>
Reviewed-by: Murtadha Hubail <[email protected]>
---
.../test/common/CancellationTestExecutor.java | 7 +-
.../asterix/test/common/ProfilingTestExecutor.java | 9 +-
.../apache/asterix/test/common/TestConstants.java | 3 +
.../apache/asterix/test/common/TestExecutor.java | 37 ++-
.../asterix/test/dataflow/LogMarkerTest.java | 2 +
.../aws/AwsS3ExternalDatasetTest.java | 7 +-
.../AzureBlobStorageExternalDatasetTest.java | 24 +-
.../copy-to/csv/null/null.03.update.sqlpp | 9 +-
.../copy-to/csv/null/null.04.ddl.sqlpp | 10 +-
.../empty-path/empty-path.01.container.sqlpp | 2 +-
.../empty-path/empty-path.02.container.sqlpp | 2 +-
.../copy-to/empty-path/empty-path.03.ddl.sqlpp | 9 +-
.../copy-to/empty-path/empty-path.04.update.sqlpp | 18 +-
.../copy-to/empty-path/empty-path.05.ddl.sqlpp | 18 +-
.../negative/long-path/long-path.01.ddl.sqlpp | 9 +-
.../negative/long-path/long-path.02.update.sqlpp | 9 +-
.../negative/long-path/long-path.03.query.sqlpp | 9 +-
.../runtime-missing/runtime-missing.01.ddl.sqlpp | 9 +-
.../runtime-missing/runtime-missing.02.query.sqlpp | 9 +-
.../runtime-missing/runtime-missing.03.query.sqlpp | 9 +-
.../runtime-missing/runtime-missing.04.ddl.sqlpp | 9 +-
.../supported-adapters.02.update.sqlpp | 35 ---
.../queries_sqlpp/copy-to/query/query.01.ddl.sqlpp | 9 +-
.../copy-to/query/query.02.update.sqlpp | 9 +-
.../queries_sqlpp/copy-to/query/query.03.ddl.sqlpp | 9 +-
...stsuite_external_dataset_azure_blob_storage.xml | 327 +++++++++++++++++++++
.../runtimets/testsuite_external_dataset_s3.xml | 21 +-
.../resources/runtimets/testsuite_sqlpp_hdfs.xml | 1 -
.../blobstorage/AzBlobStorageClientConfig.java | 24 ++
.../blobstorage/AzBlobStorageCloudClient.java | 33 +--
.../AbstractCloudExternalFileWriterFactory.java | 7 +-
.../cloud/writer/AzureExternalFileWriter.java | 51 ++++
...ry.java => AzureExternalFileWriterFactory.java} | 34 ++-
.../cloud/writer/GCSExternalFileWriterFactory.java | 11 +
.../cloud/writer/S3ExternalFileWriterFactory.java | 11 +
.../external/util/ExternalDataConstants.java | 3 +-
.../asterix/external/util/aws/s3/S3Constants.java | 3 +
.../util/azure/blob_storage/AzureConstants.java | 3 +
.../external/util/google/gcs/GCSConstants.java | 3 +
.../metadata/provider/ExternalWriterProvider.java | 2 +
40 files changed, 615 insertions(+), 201 deletions(-)
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/CancellationTestExecutor.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/CancellationTestExecutor.java
index b4b73528e8..73f0606660 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/CancellationTestExecutor.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/CancellationTestExecutor.java
@@ -51,8 +51,9 @@ public class CancellationTestExecutor extends TestExecutor {
@Override
public InputStream executeQueryService(String str,
TestCaseContext.OutputFormat fmt, URI uri,
- List<TestCase.CompilationUnit.Parameter> params, boolean
jsonEncoded, Charset responseCharset,
- Predicate<Integer> responseCodeValidator, boolean cancellable)
throws Exception {
+ List<TestCase.CompilationUnit.Parameter> params,
List<TestCase.CompilationUnit.Placeholder> placeholders,
+ boolean jsonEncoded, Charset responseCharset, Predicate<Integer>
responseCodeValidator, boolean cancellable)
+ throws Exception {
cancellable = cancellable && !containsClientContextID(str);
String clientContextId = UUID.randomUUID().toString();
final List<TestCase.CompilationUnit.Parameter> newParams =
@@ -60,7 +61,7 @@ public class CancellationTestExecutor extends TestExecutor {
Callable<InputStream> query = () -> {
try {
return CancellationTestExecutor.super.executeQueryService(str,
fmt, uri,
- constructQueryParameters(str, fmt, newParams),
jsonEncoded, responseCharset,
+ constructQueryParameters(str, fmt, newParams),
placeholders, jsonEncoded, responseCharset,
responseCodeValidator, true);
} catch (Exception e) {
e.printStackTrace();
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ProfilingTestExecutor.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ProfilingTestExecutor.java
index 44502daa0c..c08be3a0e5 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ProfilingTestExecutor.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ProfilingTestExecutor.java
@@ -33,14 +33,15 @@ public class ProfilingTestExecutor extends TestExecutor {
private final TestCase.CompilationUnit.Parameter profile = new
TestCase.CompilationUnit.Parameter();
public InputStream executeQueryService(String str,
TestCaseContext.OutputFormat fmt, URI uri,
- List<TestCase.CompilationUnit.Parameter> params, boolean
jsonEncoded, Charset responseCharset,
- Predicate<Integer> responseCodeValidator, boolean cancellable)
throws Exception {
+ List<TestCase.CompilationUnit.Parameter> params,
List<TestCase.CompilationUnit.Placeholder> placeholders,
+ boolean jsonEncoded, Charset responseCharset, Predicate<Integer>
responseCodeValidator, boolean cancellable)
+ throws Exception {
profile.setName("profile");
profile.setValue("timings");
profile.setType(ParameterTypeEnum.STRING);
params.add(profile);
- return super.executeQueryService(str, fmt, uri,
constructQueryParameters(str, fmt, params), jsonEncoded,
- responseCharset, responseCodeValidator, cancellable);
+ return super.executeQueryService(str, fmt, uri,
constructQueryParameters(str, fmt, params), placeholders,
+ jsonEncoded, responseCharset, responseCodeValidator,
cancellable);
}
}
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java
index bef2f97851..8a6b99c446 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java
@@ -85,6 +85,9 @@ public class TestConstants {
+ "(\"accountKey\"=\"" + AZURITE_ACCOUNT_KEY_DEFAULT +
"\"),\n" + "(\"endpoint\"=\""
+ BLOB_ENDPOINT_PLACEHOLDER + "\")";
public static final String TEMPLATE_DEFAULT = TEMPLATE;
+ public static final String TEMPLATE_DEFAULT_NO_PARENTHESES_WITH_COLONS
=
+ "\"accountName\":\"" + AZURITE_ACCOUNT_NAME_DEFAULT + "\",\n"
+ "\"accountKey\":\""
+ + AZURITE_ACCOUNT_KEY_DEFAULT + "\",\n" +
"\"endpoint\":\"" + BLOB_ENDPOINT_PLACEHOLDER + "\"";
}
public static class HDFS {
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index 3d39ef4539..f9d0efb1d2 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -933,9 +933,9 @@ public class TestExecutor {
}
public InputStream executeQueryService(String str, TestFileContext ctx,
OutputFormat fmt, URI uri,
- List<Parameter> params, boolean jsonEncoded, Charset
responseCharset,
+ List<Parameter> params, List<Placeholder> placeholders, boolean
jsonEncoded, Charset responseCharset,
Predicate<Integer> responseCodeValidator, boolean cancellable)
throws Exception {
- return executeQueryService(str, fmt, uri,
constructQueryParameters(str, fmt, params), jsonEncoded,
+ return executeQueryService(str, fmt, uri,
constructQueryParameters(str, fmt, params), placeholders, jsonEncoded,
responseCharset, responseCodeValidator, cancellable);
}
@@ -1311,9 +1311,10 @@ public class TestExecutor {
:
expectedResultFileCtxs.get(queryCount.intValue()).getFile();
File actualResultFile = expectedResultFile == null ? null
: testCaseCtx.getActualResultFile(cUnit,
expectedResultFile, new File(actualPath));
- ExtractedResult extractedResult =
executeQuery(OutputFormat.forCompilationUnit(cUnit), statement,
- variableCtx, ctx, expectedResultFile,
actualResultFile, queryCount,
- expectedResultFileCtxs.size(), cUnit.getParameter(),
ComparisonEnum.TEXT);
+ ExtractedResult extractedResult =
+ executeQuery(OutputFormat.forCompilationUnit(cUnit),
statement, variableCtx, ctx,
+ expectedResultFile, actualResultFile,
queryCount, expectedResultFileCtxs.size(),
+ cUnit.getParameter(), cUnit.getPlaceholder(),
ComparisonEnum.TEXT);
validateWarning(extractedResult, testCaseCtx, cUnit, testFile);
break;
@@ -1327,7 +1328,7 @@ public class TestExecutor {
+ cUnit.getName() + '.' + ctx.getSeqNum() + ".adm");
executeQuery(OutputFormat.forCompilationUnit(cUnit),
statement, variableCtx, ctx, null,
actualResultFile, queryCount,
expectedResultFileCtxs.size(), cUnit.getParameter(),
- ComparisonEnum.TEXT);
+ cUnit.getPlaceholder(), ComparisonEnum.TEXT);
variableCtx.put(key, actualResultFile);
break;
case "validate":
@@ -1407,7 +1408,7 @@ public class TestExecutor {
+ " (start <test server name> <port>
[<arg1>][<arg2>][<arg3>]...");
}
String name = command[1];
- Integer port = new Integer(command[2]);
+ Integer port = Integer.valueOf(command[2]);
if (runningTestServers.containsKey(port)) {
throw new Exception("server with port " + port + "
is already running");
}
@@ -1423,7 +1424,7 @@ public class TestExecutor {
}
runningTestServers.clear();
} else {
- Integer port = new Integer(command[1]);
+ Integer port = Integer.valueOf(command[1]);
ITestServer server = runningTestServers.get(port);
if (server == null) {
throw new Exception("no server is listening to
port " + port);
@@ -1569,7 +1570,7 @@ public class TestExecutor {
+ cUnit.getName() + '.' + ctx.getSeqNum() + ".adm");
executeQuery(OutputFormat.forCompilationUnit(cUnit), statement,
variableCtx,
new TestFileContext(testFile, "validate"), expectedResultFile,
actualResultFile, queryCount,
- expectedResultFileCtxs.size(), cUnit.getParameter(),
ComparisonEnum.TEXT);
+ expectedResultFileCtxs.size(), cUnit.getParameter(),
cUnit.getPlaceholder(), ComparisonEnum.TEXT);
}
protected void executeHttpRequest(OutputFormat fmt, String statement,
Map<String, Object> variableCtx,
@@ -1642,7 +1643,8 @@ public class TestExecutor {
public ExtractedResult executeQuery(OutputFormat fmt, String statement,
Map<String, Object> variableCtx,
TestFileContext ctx, File expectedResultFile, File
actualResultFile, MutableInt queryCount,
- int numResultFiles, List<Parameter> params, ComparisonEnum
compare, URI uri) throws Exception {
+ int numResultFiles, List<Parameter> params, List<Placeholder>
placeholders, ComparisonEnum compare, URI uri)
+ throws Exception {
String delivery = DELIVERY_IMMEDIATE;
String reqType = ctx.getType();
@@ -1660,8 +1662,8 @@ public class TestExecutor {
final String variablesReplaced = replaceVarRefRelaxed(statement,
variableCtx);
String resultVar = getResultVariable(statement); //Is the result of
the statement/query to be used in later tests
if (DELIVERY_IMMEDIATE.equals(delivery)) {
- resultStream = executeQueryService(variablesReplaced, ctx, fmt,
uri, params, isJsonEncoded, responseCharset,
- null, isCancellable(reqType));
+ resultStream = executeQueryService(variablesReplaced, ctx, fmt,
uri, params, placeholders, isJsonEncoded,
+ responseCharset, null, isCancellable(reqType));
switch (reqType) {
case METRICS_QUERY_TYPE:
resultStream =
ResultExtractor.extractMetrics(resultStream, responseCharset);
@@ -1724,10 +1726,11 @@ public class TestExecutor {
public ExtractedResult executeQuery(OutputFormat fmt, String statement,
Map<String, Object> variableCtx,
TestFileContext ctx, File expectedResultFile, File
actualResultFile, MutableInt queryCount,
- int numResultFiles, List<Parameter> params, ComparisonEnum
compare) throws Exception {
+ int numResultFiles, List<Parameter> params, List<Placeholder>
placeholders, ComparisonEnum compare)
+ throws Exception {
URI uri = getEndpoint(Servlets.QUERY_SERVICE,
FilenameUtils.getExtension(ctx.getFile().getName()));
return executeQuery(fmt, statement, variableCtx, ctx,
expectedResultFile, actualResultFile, queryCount,
- numResultFiles, params, compare, uri);
+ numResultFiles, params, placeholders, compare, uri);
}
private void polldynamic(TestCaseContext testCaseCtx, TestFileContext ctx,
Map<String, Object> variableCtx,
@@ -2554,7 +2557,11 @@ public class TestExecutor {
}
protected String setAzureTemplateDefault(String str) {
- return str.replace("%template%", TEMPLATE_DEFAULT);
+ if (str.contains("%template%")) {
+ return str.replace("%template%", TEMPLATE_DEFAULT);
+ } else {
+ return str.replace("%template_colons%",
TestConstants.Azure.TEMPLATE_DEFAULT_NO_PARENTHESES_WITH_COLONS);
+ }
}
protected String setGCSTemplateDefault(String str) {
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LogMarkerTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LogMarkerTest.java
index 0de0539d4f..e6bb74d0b3 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LogMarkerTest.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LogMarkerTest.java
@@ -65,8 +65,10 @@ import
org.apache.hyracks.storage.am.lsm.common.util.ComponentUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
+@Ignore
public class LogMarkerTest {
private static final IAType[] KEY_TYPES = { BuiltinType.AINT32 };
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
index 665b8057ec..e1da469dc0 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
@@ -480,13 +480,16 @@ public class AwsS3ExternalDatasetTest {
String statement, boolean isDmlRecoveryTest, ProcessBuilder
pb, TestCase.CompilationUnit cUnit,
MutableInt queryCount, List<TestFileContext>
expectedResultFileCtxs, File testFile, String actualPath)
throws Exception {
+ statement = applyExternalDatasetSubstitution(statement,
cUnit.getPlaceholder());
String[] lines;
+ String lastLine;
+ String[] command;
switch (ctx.getType()) {
case "container":
// <bucket> <def>
<sub-path:new_fname:src_file1,sub-path:new_fname:src_file2,sub-path:src_file3>
lines =
TestExecutor.stripAllComments(statement).trim().split("\n");
- String lastLine = lines[lines.length - 1];
- String[] command = lastLine.trim().split(" ");
+ lastLine = lines[lines.length - 1];
+ command = lastLine.trim().split(" ");
int length = command.length;
if (length == 1) {
createBucket(command[0]);
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java
index d35440e4ff..675ea4e088 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java
@@ -372,6 +372,7 @@ public class AzureBlobStorageExternalDatasetTest {
String statement, boolean isDmlRecoveryTest, ProcessBuilder
pb, TestCase.CompilationUnit cUnit,
MutableInt queryCount, List<TestFileContext>
expectedResultFileCtxs, File testFile, String actualPath)
throws Exception {
+ statement = applyExternalDatasetSubstitution(statement,
cUnit.getPlaceholder());
String[] lines;
switch (ctx.getType()) {
case "container":
@@ -380,10 +381,13 @@ public class AzureBlobStorageExternalDatasetTest {
String lastLine = lines[lines.length - 1];
String[] command = lastLine.trim().split(" ");
int length = command.length;
- if (length != 3) {
- throw new Exception("invalid create container format");
+ if (length == 1) {
+ createBucket(command[0]);
+ } else if (length == 3) {
+ dropRecreateContainer(command[0], command[1],
command[2]);
+ } else {
+ throw new Exception("invalid create bucket format");
}
- dropRecreateContainer(command[0], command[1], command[2]);
break;
default:
super.executeTestFile(testCaseCtx, ctx, variableCtx,
statement, isDmlRecoveryTest, pb, cUnit,
@@ -393,6 +397,17 @@ public class AzureBlobStorageExternalDatasetTest {
}
+ private static void createBucket(String bucketName) {
+ LOGGER.info("Deleting container " + bucketName);
+ try {
+ blobServiceClient.deleteBlobContainer(bucketName);
+ } catch (Exception ex) {
+ // Ignore
+ }
+ LOGGER.info("Creating container " + bucketName);
+
blobServiceClient.getBlobContainerClient(bucketName).createIfNotExists();
+ }
+
private static void dropRecreateContainer(String containerName, String
definition, String files) {
String definitionPath = definition + (definition.endsWith("/") ? "" :
"/");
String[] fileSplits = files.split(",");
@@ -407,7 +422,8 @@ public class AzureBlobStorageExternalDatasetTest {
BlobContainerClient containerClient;
LOGGER.info("Creating container " + containerName);
- containerClient = blobServiceClient.createBlobContainer(containerName);
+ containerClient =
blobServiceClient.getBlobContainerClient(containerName);
+ containerClient.createIfNotExists();
LOGGER.info("Uploading to container " + containerName + " definition "
+ definitionPath);
fileNames.clear();
for (String fileSplit : fileSplits) {
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/null/null.03.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/null/null.03.update.sqlpp
index b101ae5f75..6806073dd9 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/null/null.03.update.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/null/null.03.update.sqlpp
@@ -21,15 +21,12 @@ USE test;
COPY (
SELECT id, null name, amount, accountNumber FROM TestCollection
) toWriter
-TO S3
+TO %adapter%
PATH ("copy-to-result", "csv", "null")
AS (id bigint, name STRING, amount float, accountNumber double)
WITH {
- "accessKeyId":"dummyAccessKey",
- "secretAccessKey":"dummySecretKey",
- "region":"us-west-2",
- "serviceEndpoint":"http://127.0.0.1:8001";,
- "container":"playground",
+ %template_colons%,
+ %additionalProperties%
"format":"csv",
"delimiter":"|",
"header":"true",
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/null/null.04.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/null/null.04.ddl.sqlpp
index f3fdc90570..2c2819717c 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/null/null.04.ddl.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/null/null.04.ddl.sqlpp
@@ -19,16 +19,12 @@
USE test;
-CREATE EXTERNAL DATASET DatasetCopyNull(ColumnType) USING S3
+CREATE EXTERNAL DATASET DatasetCopyNull(ColumnType) USING %adapter%
(
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("sessionToken"="dummySessionToken"),
+ %template%,
+ %additional_Properties%,
("header"="true"),
("delimiter"="|"),
-("region"="us-west-2"),
- ("serviceEndpoint"="http://127.0.0.1:8001";),
- ("container"="playground"),
("definition"="copy-to-result/csv/null"),
("format" = "csv"),
("requireVersionChangeDetection"="false"),
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.01.container.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.01.container.sqlpp
index 664822e9b3..a750d01f52 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.01.container.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.01.container.sqlpp
@@ -17,4 +17,4 @@
* under the License.
*/
// create empty bucket
-empty-bucket1
\ No newline at end of file
+emptybucket1
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.02.container.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.02.container.sqlpp
index eb929081f7..3e557880a0 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.02.container.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.02.container.sqlpp
@@ -17,4 +17,4 @@
* under the License.
*/
// create empty bucket
-empty-bucket2
\ No newline at end of file
+emptybucket2
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.03.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.03.ddl.sqlpp
index b68c38b195..9595138f07 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.03.ddl.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.03.ddl.sqlpp
@@ -24,12 +24,9 @@ USE test;
CREATE TYPE OpenType AS {
};
-CREATE EXTERNAL DATASET Customer(OpenType) USING S3 (
- ("accessKeyId"="dummyAccessKey"),
- ("secretAccessKey"="dummySecretKey"),
- ("region"="us-west-2"),
- ("serviceEndpoint"="http://127.0.0.1:8001";),
- ("container"="playground"),
+CREATE EXTERNAL DATASET Customer(OpenType) USING %adapter% (
+ %template%,
+ %additional_Properties%,
("definition"="external-filter/car/{company:string}/customer/{customer_id:int}"),
("embed-filter-values" = "false"),
("format"="json")
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.04.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.04.update.sqlpp
index 8ff9deb3cb..d661e30866 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.04.update.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.04.update.sqlpp
@@ -19,26 +19,20 @@
USE test;
COPY Customer c
-TO S3
+TO %adapter%
PATH ()
WITH {
- "accessKeyId":"dummyAccessKey",
- "secretAccessKey":"dummySecretKey",
- "region":"us-west-2",
- "serviceEndpoint":"http://127.0.0.1:8001";,
- "container":"empty-bucket1",
+ %template_colons%,
+ "container":"emptybucket1",
"format":"json"
};
COPY Customer c
-TO S3
+TO %adapter%
PATH ("")
WITH {
- "accessKeyId":"dummyAccessKey",
- "secretAccessKey":"dummySecretKey",
- "region":"us-west-2",
- "serviceEndpoint":"http://127.0.0.1:8001";,
- "container":"empty-bucket2",
+ %template_colons%,
+ "container":"emptybucket2",
"format":"json"
};
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.05.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.05.ddl.sqlpp
index 54bad2f62a..555c0071ea 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.05.ddl.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/empty-path/empty-path.05.ddl.sqlpp
@@ -19,22 +19,16 @@
USE test;
-CREATE EXTERNAL DATASET CustomerCopy1(OpenType) USING S3 (
- ("accessKeyId"="dummyAccessKey"),
- ("secretAccessKey"="dummySecretKey"),
- ("region"="us-west-2"),
- ("serviceEndpoint"="http://127.0.0.1:8001";),
- ("container"="empty-bucket1"),
+CREATE EXTERNAL DATASET CustomerCopy1(OpenType) USING %adapter% (
+ %template%,
+ ("container"="emptybucket1"),
("embed-filter-values" = "false"),
("format"="json")
);
-CREATE EXTERNAL DATASET CustomerCopy2(OpenType) USING S3 (
- ("accessKeyId"="dummyAccessKey"),
- ("secretAccessKey"="dummySecretKey"),
- ("region"="us-west-2"),
- ("serviceEndpoint"="http://127.0.0.1:8001";),
- ("container"="empty-bucket2"),
+CREATE EXTERNAL DATASET CustomerCopy2(OpenType) USING %adapter% (
+ %template%,
+ ("container"="emptybucket2"),
("embed-filter-values" = "false"),
("format"="json")
);
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/long-path/long-path.01.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/long-path/long-path.01.ddl.sqlpp
index b68c38b195..9595138f07 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/long-path/long-path.01.ddl.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/long-path/long-path.01.ddl.sqlpp
@@ -24,12 +24,9 @@ USE test;
CREATE TYPE OpenType AS {
};
-CREATE EXTERNAL DATASET Customer(OpenType) USING S3 (
- ("accessKeyId"="dummyAccessKey"),
- ("secretAccessKey"="dummySecretKey"),
- ("region"="us-west-2"),
- ("serviceEndpoint"="http://127.0.0.1:8001";),
- ("container"="playground"),
+CREATE EXTERNAL DATASET Customer(OpenType) USING %adapter% (
+ %template%,
+ %additional_Properties%,
("definition"="external-filter/car/{company:string}/customer/{customer_id:int}"),
("embed-filter-values" = "false"),
("format"="json")
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/long-path/long-path.02.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/long-path/long-path.02.update.sqlpp
index ef35a36bed..f5c209e2f7 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/long-path/long-path.02.update.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/long-path/long-path.02.update.sqlpp
@@ -20,14 +20,11 @@
USE test;
COPY Customer c
-TO S3
+TO %adapter%
PATH
("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
[...]
WITH {
- "accessKeyId":"dummyAccessKey",
- "secretAccessKey":"dummySecretKey",
- "region":"us-west-2",
- "serviceEndpoint":"http://127.0.0.1:8001";,
- "container":"playground",
+ %template_colons%,
+ %additionalProperties%
"format":"json"
}
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/long-path/long-path.03.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/long-path/long-path.03.query.sqlpp
index e5cc591f28..f6db11864c 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/long-path/long-path.03.query.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/long-path/long-path.03.query.sqlpp
@@ -25,17 +25,14 @@ COPY (
-- Minimize the number of warnings
WHERE c.company = "ford"
) toWrite
-TO S3
+TO %adapter%
PATH (company,
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
[...]
OVER (
PARTITION BY toWrite.company company
)
WITH {
- "accessKeyId":"dummyAccessKey",
- "secretAccessKey":"dummySecretKey",
- "region":"us-west-2",
- "serviceEndpoint":"http://127.0.0.1:8001";,
- "container":"playground",
+ %template_colons%,
+ %additionalProperties%
"format":"json"
}
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/runtime-missing/runtime-missing.01.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/runtime-missing/runtime-missing.01.ddl.sqlpp
index b68c38b195..9595138f07 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/runtime-missing/runtime-missing.01.ddl.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/runtime-missing/runtime-missing.01.ddl.sqlpp
@@ -24,12 +24,9 @@ USE test;
CREATE TYPE OpenType AS {
};
-CREATE EXTERNAL DATASET Customer(OpenType) USING S3 (
- ("accessKeyId"="dummyAccessKey"),
- ("secretAccessKey"="dummySecretKey"),
- ("region"="us-west-2"),
- ("serviceEndpoint"="http://127.0.0.1:8001";),
- ("container"="playground"),
+CREATE EXTERNAL DATASET Customer(OpenType) USING %adapter% (
+ %template%,
+ %additional_Properties%,
("definition"="external-filter/car/{company:string}/customer/{customer_id:int}"),
("embed-filter-values" = "false"),
("format"="json")
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/runtime-missing/runtime-missing.02.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/runtime-missing/runtime-missing.02.query.sqlpp
index 158ea04837..d94c57ae27 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/runtime-missing/runtime-missing.02.query.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/runtime-missing/runtime-missing.02.query.sqlpp
@@ -20,17 +20,14 @@
USE test;
COPY Customer c
-TO S3
+TO %adapter%
PATH ("copy-to-result", myMissingValue)
OVER (
PARTITION BY c.doesNotExist myMissingValue
)
WITH {
- "accessKeyId":"dummyAccessKey",
- "secretAccessKey":"dummySecretKey",
- "region":"us-west-2",
- "serviceEndpoint":"http://127.0.0.1:8001";,
- "container":"playground",
+ %template_colons%,
+ %additionalProperties%
"format":"json",
"compression":"gzip"
}
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/runtime-missing/runtime-missing.03.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/runtime-missing/runtime-missing.03.query.sqlpp
index 0dfb3d33c5..4685a4e5f4 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/runtime-missing/runtime-missing.03.query.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/runtime-missing/runtime-missing.03.query.sqlpp
@@ -27,17 +27,14 @@ COPY (
c.year
FROM Customer c
) toWrite
-TO S3
+TO %adapter%
PATH ("copy-to-result/someMissing", company)
OVER (
PARTITION BY toWrite.company company
)
WITH {
- "accessKeyId":"dummyAccessKey",
- "secretAccessKey":"dummySecretKey",
- "region":"us-west-2",
- "serviceEndpoint":"http://127.0.0.1:8001";,
- "container":"playground",
+ %template_colons%,
+ %additionalProperties%
"format":"json"
}
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/runtime-missing/runtime-missing.04.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/runtime-missing/runtime-missing.04.ddl.sqlpp
index 5277555ce1..f3e59bbecb 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/runtime-missing/runtime-missing.04.ddl.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/runtime-missing/runtime-missing.04.ddl.sqlpp
@@ -19,12 +19,9 @@
USE test;
-CREATE EXTERNAL DATASET CustomerCopy(OpenType) USING S3 (
- ("accessKeyId"="dummyAccessKey"),
- ("secretAccessKey"="dummySecretKey"),
- ("region"="us-west-2"),
- ("serviceEndpoint"="http://127.0.0.1:8001";),
- ("container"="playground"),
+CREATE EXTERNAL DATASET CustomerCopy(OpenType) USING %adapter% (
+ %template%,
+ %additional_Properties%,
("definition"="copy-to-result/someMissing/{company:string}"),
("embed-filter-values" = "false"),
("format"="json")
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/supported-adapter-format-compression/supported-adapters.02.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/supported-adapter-format-compression/supported-adapters.02.update.sqlpp
deleted file mode 100644
index b016330747..0000000000
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/supported-adapter-format-compression/supported-adapters.02.update.sqlpp
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-USE test;
-
-COPY Customer c
-TO AZUREBLOB
-PATH ("copy-to-result")
-WITH {
- "accessKeyId":"dummyAccessKey",
- "secretAccessKey":"dummySecretKey",
- "region":"us-west-2",
- "serviceEndpoint":"http://127.0.0.1:8001";,
- "container":"playground",
- "format":"json"
-}
-
-
-
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/query/query.01.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/query/query.01.ddl.sqlpp
index b68c38b195..9595138f07 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/query/query.01.ddl.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/query/query.01.ddl.sqlpp
@@ -24,12 +24,9 @@ USE test;
CREATE TYPE OpenType AS {
};
-CREATE EXTERNAL DATASET Customer(OpenType) USING S3 (
- ("accessKeyId"="dummyAccessKey"),
- ("secretAccessKey"="dummySecretKey"),
- ("region"="us-west-2"),
- ("serviceEndpoint"="http://127.0.0.1:8001";),
- ("container"="playground"),
+CREATE EXTERNAL DATASET Customer(OpenType) USING %adapter% (
+ %template%,
+ %additional_Properties%,
("definition"="external-filter/car/{company:string}/customer/{customer_id:int}"),
("embed-filter-values" = "false"),
("format"="json")
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/query/query.02.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/query/query.02.update.sqlpp
index 1d7253b21b..8e5817190c 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/query/query.02.update.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/query/query.02.update.sqlpp
@@ -23,18 +23,15 @@ COPY (
SELECT DISTINCT UPPERCASE(c.company) company, c.year
FROM Customer c
) toWriter
-TO S3
+TO %adapter%
PATH ("copy-to-result", "company-year", company, year)
OVER (
PARTITION BY toWriter.company company,
toWriter.year year
)
WITH {
- "accessKeyId":"dummyAccessKey",
- "secretAccessKey":"dummySecretKey",
- "region":"us-west-2",
- "serviceEndpoint":"http://127.0.0.1:8001";,
- "container":"playground",
+ %template_colons%,
+ %additionalProperties%
"format":"json",
"compression":"gzip",
"gzipCompressionLevel": "1"
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/query/query.03.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/query/query.03.ddl.sqlpp
index 6c19d01150..4698365b27 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/query/query.03.ddl.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/query/query.03.ddl.sqlpp
@@ -19,12 +19,9 @@
USE test;
-CREATE EXTERNAL DATASET CustomerCopy(OpenType) USING S3 (
- ("accessKeyId"="dummyAccessKey"),
- ("secretAccessKey"="dummySecretKey"),
- ("region"="us-west-2"),
- ("serviceEndpoint"="http://127.0.0.1:8001";),
- ("container"="playground"),
+CREATE EXTERNAL DATASET CustomerCopy(OpenType) USING %adapter% (
+ %template%,
+ %additional_Properties%,
("definition"="copy-to-result/company-year/{company:string}/{year:int}"),
("embed-filter-values" = "false"),
("format"="json")
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
index b801847ae0..ac56d9c7ec 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
@@ -18,6 +18,333 @@
! under the License.
!-->
<test-suite xmlns="urn:xml.testframework.asterix.apache.org"
ResultOffsetPath="results" QueryOffsetPath="queries_sqlpp"
QueryFileExtension=".sqlpp">
+ <test-group name="copy-to">
+ <test-case FilePath="copy-to">
+ <compilation-unit name="partition">
+ <placeholder name="adapter" value="AZUREBLOB" />
+ <placeholder name="pathprefix" value="" />
+ <placeholder name="path_prefix" value="" />
+ <placeholder name="additionalProperties"
value='"container":"playground",' />
+ <placeholder name="additional_Properties"
value='("container"="playground")' />
+ <output-dir compare="Text">partition</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="copy-to">
+ <compilation-unit name="simple-write">
+ <placeholder name="adapter" value="AZUREBLOB" />
+ <placeholder name="pathprefix" value="" />
+ <placeholder name="path_prefix" value="" />
+ <placeholder name="additionalProperties"
value='"container":"playground",' />
+ <placeholder name="additional_Properties"
value='("container"="playground")' />
+ <output-dir compare="Text">simple-write</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="copy-to">
+ <compilation-unit name="query">
+ <placeholder name="adapter" value="AZUREBLOB" />
+ <placeholder name="pathprefix" value="" />
+ <placeholder name="path_prefix" value="" />
+ <placeholder name="additionalProperties"
value='"container":"playground",' />
+ <placeholder name="additional_Properties"
value='("container"="playground")' />
+ <output-dir compare="Text">query</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="copy-to">
+ <compilation-unit name="default-namespace">
+ <placeholder name="adapter" value="AZUREBLOB" />
+ <placeholder name="pathprefix" value="" />
+ <placeholder name="path_prefix" value="" />
+ <placeholder name="additionalProperties"
value='"container":"playground",' />
+ <placeholder name="additional_Properties"
value='("container"="playground")' />
+ <output-dir compare="Text">default-namespace</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!-- Parquet writing doesn't work with Azurite emulator
+ <test-case FilePath="copy-to">
+ <compilation-unit name="parquet-simple">
+ <placeholder name="adapter" value="AZUREBLOB" />
+ <placeholder name="pathprefix" value="" />
+ <placeholder name="path_prefix" value="" />
+ <placeholder name="additionalProperties"
value='"container":"playground",' />
+ <placeholder name="additional_Properties"
value='("container"="playground")' />
+ <output-dir compare="Text">parquet-simple</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="copy-to">
+ <compilation-unit name="parquet-tweet">
+ <placeholder name="adapter" value="AZUREBLOB" />
+ <placeholder name="pathprefix" value="" />
+ <placeholder name="path_prefix" value="" />
+ <placeholder name="additionalProperties"
value='"container":"playground",' />
+ <placeholder name="additional_Properties"
value='("container"="playground")' />
+ <output-dir compare="Text">parquet-tweet</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="copy-to">
+ <compilation-unit name="parquet-partition-heterogeneous">
+ <placeholder name="adapter" value="AZUREBLOB" />
+ <placeholder name="pathprefix" value="" />
+ <placeholder name="path_prefix" value="" />
+ <placeholder name="additionalProperties"
value='"container":"playground",' />
+ <placeholder name="additional_Properties"
value='("container"="playground")' />
+ <output-dir compare="Text">parquet-partition-heterogeneous</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="copy-to">
+ <compilation-unit name="parquet-utf8">
+ <placeholder name="adapter" value="AZUREBLOB" />
+ <placeholder name="pathprefix" value="" />
+ <placeholder name="path_prefix" value="" />
+ <placeholder name="additionalProperties"
value='"container":"playground",' />
+ <placeholder name="additional_Properties"
value='("container"="playground")' />
+ <output-dir compare="Text">parquet-utf8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="copy-to">
+ <compilation-unit name="parquet-heterogeneous">
+ <placeholder name="adapter" value="AZUREBLOB" />
+ <placeholder name="pathprefix" value="" />
+ <placeholder name="path_prefix" value="" />
+ <placeholder name="additionalProperties"
value='"container":"playground",' />
+ <placeholder name="additional_Properties"
value='("container"="playground")' />
+ <output-dir compare="Text">parquet-heterogeneous</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="copy-to">
+ <compilation-unit name="parquet-cover-data-types">
+ <placeholder name="adapter" value="AZUREBLOB" />
+ <placeholder name="pathprefix" value="" />
+ <placeholder name="path_prefix" value="" />
+ <placeholder name="additionalProperties"
value='"container":"playground",' />
+ <placeholder name="additional_Properties"
value='("container"="playground")' />
+ <output-dir compare="Text">parquet-cover-data-types</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="copy-to">
+ <compilation-unit name="parquet-field-names">
+ <placeholder name="adapter" value="AZUREBLOB" />
+ <placeholder name="pathprefix" value="" />
+ <placeholder name="path_prefix" value="" />
+ <placeholder name="additionalProperties"
value='"container":"playground",' />
+ <placeholder name="additional_Properties"
value='("container"="playground")' />
+ <output-dir compare="Text">parquet-field-names</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="copy-to">
+ <compilation-unit name="parquet-empty-array">
+ <placeholder name="adapter" value="AZUREBLOB" />
+ <placeholder name="pathprefix" value="" />
+ <placeholder name="path_prefix" value="" />
+ <placeholder name="additionalProperties"
value='"container":"playground",' />
+ <placeholder name="additional_Properties"
value='("container"="playground")' />
+ <output-dir compare="Text">parquet-empty-array</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ <test-case FilePath="copy-to">
+ <compilation-unit name="empty-path">
+ <placeholder name="adapter" value="AZUREBLOB" />
+ <placeholder name="pathprefix" value="" />
+ <placeholder name="path_prefix" value="" />
+ <placeholder name="additionalProperties"
value='"container":"playground",' />
+ <placeholder name="additional_Properties"
value='("container"="playground")' />
+ <output-dir compare="Text">empty-path</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="copy-to">
+ <compilation-unit name="order-by">
+ <placeholder name="adapter" value="AZUREBLOB" />
+ <placeholder name="pathprefix" value="" />
+ <placeholder name="path_prefix" value="" />
+ <placeholder name="additionalProperties"
value='"container":"playground",' />
+ <placeholder name="additional_Properties"
value='("container"="playground")' />
+ <output-dir compare="Text">order-by</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="copy-to/negative">
+ <compilation-unit name="early-missing">
+ <placeholder name="adapter" value="AZUREBLOB" />
+ <placeholder name="path_prefix" value="" />
+ <placeholder name="additionalProperties"
value='"container":"playground",' />
+ <placeholder name="additional_Properties"
value='("container"="playground")' />
+ <output-dir compare="Text">early-missing</output-dir>
+ <expected-error>ASX0064: Path expression produced a value of type
'missing'. Path must be of type string</expected-error>
+ <expected-error>ASX0064: Path expression produced a value of type
'null'. Path must be of type string</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="copy-to/negative" check-warnings="true">
+ <compilation-unit name="long-path">
+ <placeholder name="adapter" value="AZUREBLOB" />
+ <placeholder name="pathprefix" value="" />
+ <placeholder name="path_prefix" value="" />
+ <placeholder name="additionalProperties"
value='"container":"playground",' />
+ <placeholder name="additional_Properties"
value='("container"="playground")' />
+ <output-dir compare="Text">long-path</output-dir>
+ <expected-error>ASX0065: Length of the file path
'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
[...]
+ <expected-warn>ASX0065: Length of the file path
'ford/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
[...]
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="copy-to/negative">
+ <compilation-unit name="non-empty-folder">
+ <output-dir compare="Text">non-empty-folder</output-dir>
+ <placeholder name="adapter" value="AZUREBLOB" />
+ <placeholder name="pathprefix" value="" />
+ <placeholder name="path_prefix" value="" />
+ <placeholder name="additionalProperties"
value='"container":"playground",' />
+ <placeholder name="additional_Properties"
value='("container"="playground")' />
+ <expected-error>ASX0062: Cannot write to a non-empty directory
'copy-to-result/duplicate-write'</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="copy-to/negative" check-warnings="true">
+ <compilation-unit name="runtime-missing">
+ <placeholder name="adapter" value="AZUREBLOB" />
+ <placeholder name="pathprefix" value="" />
+ <placeholder name="path_prefix" value="" />
+ <placeholder name="additionalProperties"
value='"container":"playground",' />
+ <placeholder name="additional_Properties"
value='("container"="playground")' />
+ <output-dir compare="Text">runtime-missing</output-dir>
+ <expected-warn>ASX0064: Path expression produced a value of type
'missing'. Path must be of type string (in line 24, at column 7)</expected-warn>
+ <expected-warn>ASX0064: Path expression produced a value of type
'missing'. Path must be of type string (in line 31, at column 7)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="copy-to/negative">
+ <compilation-unit name="supported-adapter-format-compression">
+ <placeholder name="cleanprefix" value="playground copy-to-result" />
+ <placeholder name="adapter" value="AZUREBLOB" />
+ <placeholder name="pathprefix" value="" />
+ <placeholder name="path_prefix" value="" />
+ <placeholder name="additionalProperties"
value='"container":"playground",' />
+ <placeholder name="additional_Properties"
value='("container"="playground")' />
+ <output-dir
compare="Text">supported-adapter-format-compression</output-dir>
+ <expected-error>ASX1189: Unsupported writing format 'avro'. Supported
formats: [csv, json, parquet]</expected-error>
+ <expected-error>ASX1202: Unsupported compression scheme rar. Supported
schemes for json are [gzip]</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="copy-to/negative">
+ <compilation-unit name="parquet-error-checks">
+ <placeholder name="adapter" value="AZUREBLOB" />
+ <placeholder name="pathprefix" value="" />
+ <placeholder name="path_prefix" value="" />
+ <placeholder name="additionalProperties"
value='"container":"playground",' />
+ <placeholder name="additional_Properties"
value='("container"="playground")' />
+ <output-dir compare="Text">parquet-error-checks</output-dir>
+ <expected-error>ASX0037: Type mismatch: expected value of type BINARY,
but got the value of type bigint</expected-error>
+ <expected-error>HYR0132: Extra field in the result, field 'second'
does not exist at 'nested' in the schema</expected-error>
+ <expected-error>HYR0131: Result does not follow the schema, group type
expected but found primitive type at 'nested'</expected-error>
+ <expected-error>HYR0131: Result does not follow the schema, primitive
type expected but found group type at 'name'</expected-error>
+ <expected-error>ASX1206: Storage units expected for the field
'row-group-size' (e.g., 0.1KB, 100kb, 1mb, 3MB, 8.5GB ...). Provided
'random'</expected-error>
+ <expected-error>ASX1206: Storage units expected for the field
'page-size' (e.g., 0.1KB, 100kb, 1mb, 3MB, 8.5GB ...). Provided
'random'</expected-error>
+ <expected-error>ASX1202: Unsupported compression scheme rar. Supported
schemes for parquet are [gzip, snappy, zstd]</expected-error>
+ <expected-error>ASX1001: Syntax error</expected-error>
+ <expected-error>ASX1204: 'binary' type not supported in parquet
format</expected-error>
+ <expected-error>ASX1205: Invalid Parquet Writer Version provided '3'.
Supported values: [1, 2]</expected-error>
+ <expected-error>ASX0039: Expected integer value, got yvghc (in line
22, at column 6)</expected-error>
+ <expected-error>ASX1209: Maximum value allowed for 'max-schemas' is
10. Found 15</expected-error>
+ <expected-error>HYR0133: Schema could not be inferred, empty types
found in the result</expected-error>
+ <expected-error>HYR0134: Schema Limit exceeded, maximum number of
heterogeneous schemas allowed : '2'</expected-error>
+ <expected-error>ASX1204: 'rectangle' type not supported in parquet
format</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="copy-to/negative">
+ <compilation-unit name="empty-over">
+ <placeholder name="adapter" value="AZUREBLOB" />
+ <placeholder name="pathprefix" value="" />
+ <placeholder name="path_prefix" value="" />
+ <placeholder name="additionalProperties"
value='"container":"playground",' />
+ <placeholder name="additional_Properties"
value='("container"="playground")' />
+ <output-dir compare="Text">empty-over</output-dir>
+ <expected-error>ASX1001: Syntax error: OVER-clause cannot be
empty</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="copy-to/negative">
+ <compilation-unit name="bad-max-objects-per-file">
+ <placeholder name="adapter" value="AZUREBLOB" />
+ <placeholder name="pathprefix" value="" />
+ <placeholder name="path_prefix" value="" />
+ <placeholder name="additionalProperties"
value='"container":"playground",' />
+ <placeholder name="additional_Properties"
value='("container"="playground")' />
+ <output-dir compare="Text">bad-max-objects-per-file</output-dir>
+ <expected-error>Minimum value allowed for 'max-objects-per-file' is
1000. Found 2</expected-error>
+ <expected-error>Expected integer value, got hello</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="copy-to/negative">
+ <compilation-unit name="csv-error-checks">
+ <placeholder name="adapter" value="AZUREBLOB" />
+ <placeholder name="pathprefix" value="" />
+ <placeholder name="path_prefix" value="" />
+ <placeholder name="additionalProperties"
value='"container":"playground",' />
+ <placeholder name="additional_Properties"
value='("container"="playground")' />
+ <output-dir compare="Text">csv-error-checks</output-dir>
+ <expected-error>ASX1079: Compilation error: TYPE/AS Expression is
required for csv format</expected-error>
+ <expected-error>ASX1082: Cannot find datatype with name wrongDataType
(in line 27, at column 4)</expected-error>
+ <expected-error>ASX3124: 'ABCD' is not a valid quote. The length of a
quote should be 1</expected-error>
+ <expected-error>ASX3049: 'wrongDelimiter' is not a valid delimiter.
The length of a delimiter should be 1</expected-error>
+ <expected-error>ASX3126: 'wrongEscape' is not a valid escape. The
length of a escape should be 1</expected-error>
+ <expected-error>ASX3125: 'ABCD' is not a valid force-quote input. The
length of a force-quote input should be 1 character</expected-error>
+ <expected-error>ASX1207: 'object' type not supported in csv
format</expected-error>
+ <expected-error>ASX1207: 'array' type not supported in csv
format</expected-error>
+ <expected-error>Syntax error: Both 'TYPE()' and 'AS()' are provided.
Please use either 'TYPE()' or 'AS()'.</expected-error>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="copy-to/csv">
+ <test-case FilePath="copy-to/csv">
+ <compilation-unit name="simple-csv">
+ <placeholder name="adapter" value="AZUREBLOB" />
+ <placeholder name="pathprefix" value="" />
+ <placeholder name="path_prefix" value="" />
+ <placeholder name="additionalProperties"
value='"container":"playground",' />
+ <placeholder name="additional_Properties"
value='("container"="playground")' />
+ <output-dir compare="Text">simple-csv</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="copy-to/csv">
+ <compilation-unit name="quote-escape">
+ <placeholder name="adapter" value="AZUREBLOB" />
+ <placeholder name="pathprefix" value="" />
+ <placeholder name="path_prefix" value="" />
+ <placeholder name="additionalProperties"
value='"container":"playground",' />
+ <placeholder name="additional_Properties"
value='("container"="playground")' />
+ <output-dir compare="Text">quote-escape</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="copy-to/csv">
+ <compilation-unit name="delimiter">
+ <placeholder name="adapter" value="AZUREBLOB" />
+ <placeholder name="pathprefix" value="" />
+ <placeholder name="path_prefix" value="" />
+ <placeholder name="additionalProperties"
value='"container":"playground",' />
+ <placeholder name="additional_Properties"
value='("container"="playground")' />
+ <output-dir compare="Text">delimiter</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!-- These 2 tests end up writing a 0 byte file (empty), and the emulator
has a problem where it does not
+ set the contentRange value correctly in that case, leading to an NPE,
tested on Azure servers and it worked
+ fine, keeping these 2 tests disabled
+ <test-case FilePath="copy-to/csv">
+ <compilation-unit name="type-mismatch">
+ <placeholder name="adapter" value="AZUREBLOB" />
+ <placeholder name="cleanprefix" value="playground
copy-to-result/csv/type-mismatch" />
+ <placeholder name="pathprefix" value="" />
+ <placeholder name="path_prefix" value="" />
+ <placeholder name="additionalProperties"
value='"container":"playground",' />
+ <placeholder name="additional_Properties"
value='("container"="playground")' />
+ <output-dir compare="Text">type-mismatch</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="copy-to/csv">
+ <compilation-unit name="header">
+ <placeholder name="adapter" value="AZUREBLOB" />
+ <placeholder name="pathprefix" value="" />
+ <placeholder name="path_prefix" value="" />
+ <placeholder name="additionalProperties"
value='"container":"playground",' />
+ <placeholder name="additional_Properties"
value='("container"="playground")' />
+ <output-dir compare="Text">header</output-dir>
+ </compilation-unit>
+ </test-case>
+ -->
+ </test-group>
<test-group name="authentication">
<test-case
FilePath="external-dataset/azure_blob_storage/auth-methods/valid-auth-methods">
<compilation-unit name="account-name-and-account-key">
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index 95291adc6e..d23eb7a2a7 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -41,6 +41,11 @@
</test-case>
<test-case FilePath="copy-to">
<compilation-unit name="query">
+ <placeholder name="adapter" value="S3" />
+ <placeholder name="pathprefix" value="" />
+ <placeholder name="path_prefix" value="" />
+ <placeholder name="additionalProperties"
value='"container":"playground",' />
+ <placeholder name="additional_Properties"
value='("container"="playground")' />
<output-dir compare="Text">query</output-dir>
</compilation-unit>
</test-case>
@@ -136,6 +141,11 @@
</test-case>
<test-case FilePath="copy-to">
<compilation-unit name="empty-path">
+ <placeholder name="adapter" value="S3" />
+ <placeholder name="pathprefix" value="" />
+ <placeholder name="path_prefix" value="" />
+ <placeholder name="additionalProperties"
value='"container":"playground",' />
+ <placeholder name="additional_Properties"
value='("container"="playground")' />
<output-dir compare="Text">empty-path</output-dir>
</compilation-unit>
</test-case>
@@ -162,6 +172,11 @@
</test-case>
<test-case FilePath="copy-to/negative" check-warnings="true">
<compilation-unit name="long-path">
+ <placeholder name="adapter" value="S3" />
+ <placeholder name="pathprefix" value="" />
+ <placeholder name="path_prefix" value="" />
+ <placeholder name="additionalProperties"
value='"container":"playground",' />
+ <placeholder name="additional_Properties"
value='("container"="playground")' />
<output-dir compare="Text">long-path</output-dir>
<expected-error>ASX0065: Length of the file path
'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
[...]
<expected-warn>ASX0065: Length of the file path
'ford/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
[...]
@@ -180,6 +195,11 @@
</test-case>
<test-case FilePath="copy-to/negative" check-warnings="true">
<compilation-unit name="runtime-missing">
+ <placeholder name="adapter" value="S3" />
+ <placeholder name="pathprefix" value="" />
+ <placeholder name="path_prefix" value="" />
+ <placeholder name="additionalProperties"
value='"container":"playground",' />
+ <placeholder name="additional_Properties"
value='("container"="playground")' />
<output-dir compare="Text">runtime-missing</output-dir>
<expected-warn>ASX0064: Path expression produced a value of type
'missing'. Path must be of type string (in line 24, at column 7)</expected-warn>
<expected-warn>ASX0064: Path expression produced a value of type
'missing'. Path must be of type string (in line 31, at column 7)</expected-warn>
@@ -193,7 +213,6 @@
<placeholder name="additionalProperties"
value='"container":"playground",' />
<placeholder name="additional_Properties"
value='("container"="playground")' />
<output-dir
compare="Text">supported-adapter-format-compression</output-dir>
- <expected-error>ASX1188: Unsupported writing adapter 'AZUREBLOB'.
Supported adapters: [gcs, hdfs, localfs, s3]</expected-error>
<expected-error>ASX1189: Unsupported writing format 'avro'. Supported
formats: [csv, json, parquet]</expected-error>
<expected-error>ASX1202: Unsupported compression scheme rar. Supported
schemes for json are [gzip]</expected-error>
</compilation-unit>
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp_hdfs.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp_hdfs.xml
index f2ec232456..c3e0da0e1f 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp_hdfs.xml
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp_hdfs.xml
@@ -232,7 +232,6 @@
<placeholder name="additionalProperties" value="" />
<placeholder name="additional_Properties" value='("input-format" =
"text-input-format")' />
<output-dir
compare="Text">supported-adapter-format-compression</output-dir>
- <expected-error>ASX1188: Unsupported writing adapter 'AZUREBLOB'.
Supported adapters: [gcs, hdfs, localfs, s3]</expected-error>
<expected-error>ASX1189: Unsupported writing format 'avro'. Supported
formats: [csv, json, parquet]</expected-error>
<expected-error>ASX1202: Unsupported compression scheme rar. Supported
schemes for json are [gzip]</expected-error>
</compilation-unit>
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageClientConfig.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageClientConfig.java
index 9aedfc3254..f4536a18fe 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageClientConfig.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageClientConfig.java
@@ -18,9 +18,12 @@
*/
package org.apache.asterix.cloud.clients.azure.blobstorage;
+import java.util.Map;
import java.util.Objects;
import org.apache.asterix.common.config.CloudProperties;
+import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.asterix.external.util.azure.blob_storage.AzureConstants;
import com.azure.identity.DefaultAzureCredential;
import com.azure.identity.DefaultAzureCredentialBuilder;
@@ -40,6 +43,11 @@ public class AzBlobStorageClientConfig {
private final int writeMaxRequestsPerSeconds;
private final int readMaxRequestsPerSeconds;
+ public AzBlobStorageClientConfig(String region, String endpoint, String
prefix, boolean anonymousAuth,
+ long profilerLogInterval, String bucket, int writeBufferSize) {
+ this(region, endpoint, prefix, anonymousAuth, profilerLogInterval,
bucket, 1, 0, 0, writeBufferSize);
+ }
+
public AzBlobStorageClientConfig(String region, String endpoint, String
prefix, boolean anonymousAuth,
long profilerLogInterval, String bucket, long tokenAcquireTimeout,
int writeMaxRequestsPerSeconds,
int readMaxRequestsPerSeconds, int writeBufferSize) {
@@ -63,6 +71,22 @@ public class AzBlobStorageClientConfig {
cloudProperties.getReadMaxRequestsPerSecond(),
cloudProperties.getWriteBufferSize());
}
+ public static AzBlobStorageClientConfig of(Map<String, String>
configuration, int writeBufferSize) {
+ // Used to determine local vs. actual azure
+ String endPoint =
configuration.getOrDefault(AzureConstants.ENDPOINT_FIELD_NAME, "");
+ String bucket =
configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
+ // Disabled
+ long profilerLogInterval = 0;
+
+ // Dummy values;
+ String region = "";
+ String prefix = "";
+ boolean anonymousAuth = false;
+
+ return new AzBlobStorageClientConfig(region, endPoint, prefix,
anonymousAuth, profilerLogInterval, bucket,
+ writeBufferSize);
+ }
+
public String getRegion() {
return region;
}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java
index 0a6753478c..8e273a11ac 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java
@@ -62,7 +62,8 @@ import com.azure.core.http.rest.Response;
import com.azure.core.util.BinaryData;
import com.azure.storage.blob.BlobClient;
import com.azure.storage.blob.BlobContainerClient;
-import com.azure.storage.blob.BlobContainerClientBuilder;
+import com.azure.storage.blob.BlobServiceClient;
+import com.azure.storage.blob.BlobServiceClientBuilder;
import com.azure.storage.blob.batch.BlobBatchClient;
import com.azure.storage.blob.batch.BlobBatchClientBuilder;
import com.azure.storage.blob.models.BlobErrorCode;
@@ -85,9 +86,9 @@ public class AzBlobStorageCloudClient implements ICloudClient
{
"Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==";
private static final int SUCCESS_RESPONSE_CODE = 202;
private final ICloudGuardian guardian;
- private BlobContainerClient blobContainerClient;
- private AzBlobStorageClientConfig config;
- private IRequestProfilerLimiter profiler;
+ private final BlobContainerClient blobContainerClient;
+ private final AzBlobStorageClientConfig config;
+ private final IRequestProfilerLimiter profiler;
private final BlobBatchClient blobBatchClient;
private static final Logger LOGGER = LogManager.getLogger();
@@ -95,9 +96,10 @@ public class AzBlobStorageCloudClient implements
ICloudClient {
this(config, buildClient(config), guardian);
}
- public AzBlobStorageCloudClient(AzBlobStorageClientConfig config,
BlobContainerClient blobContainerClient,
+ public AzBlobStorageCloudClient(AzBlobStorageClientConfig config,
BlobServiceClient blobServiceClient,
ICloudGuardian guardian) {
- this.blobContainerClient = blobContainerClient;
+ this.blobContainerClient =
blobServiceClient.getBlobContainerClient(config.getBucket());
+ this.blobContainerClient.createIfNotExists();
this.config = config;
this.guardian = guardian;
long profilerInterval = config.getProfilerLogInterval();
@@ -348,8 +350,7 @@ public class AzBlobStorageCloudClient implements
ICloudClient {
profiler.objectsList();
ListBlobsOptions listBlobsOptions = new
ListBlobsOptions().setPrefix(config.getPrefix() + path);
//MAX_VALUE below represents practically no timeout
- PagedIterable<BlobItem> blobItems =
- blobContainerClient.listBlobs(listBlobsOptions,
Duration.ofDays(Long.MAX_VALUE));
+ PagedIterable<BlobItem> blobItems =
blobContainerClient.listBlobs(listBlobsOptions, Duration.ofMinutes(2));
return blobItems.stream().findAny().isEmpty();
}
@@ -391,17 +392,15 @@ public class AzBlobStorageCloudClient implements
ICloudClient {
// Hence this implementation is a no op.
}
- private static BlobContainerClient buildClient(AzBlobStorageClientConfig
config) {
- BlobContainerClientBuilder blobContainerClientBuilder =
- new
BlobContainerClientBuilder().containerName(config.getBucket()).endpoint(getEndpoint(config));
-
blobContainerClientBuilder.httpLogOptions(AzureConstants.HTTP_LOG_OPTIONS);
- configCredentialsToAzClient(blobContainerClientBuilder, config);
- BlobContainerClient blobContainerClient =
blobContainerClientBuilder.buildClient();
- blobContainerClient.createIfNotExists();
- return blobContainerClient;
+ private static BlobServiceClient buildClient(AzBlobStorageClientConfig
config) {
+ BlobServiceClientBuilder blobServiceClientBuilder = new
BlobServiceClientBuilder();
+ blobServiceClientBuilder.endpoint(getEndpoint(config));
+
blobServiceClientBuilder.httpLogOptions(AzureConstants.HTTP_LOG_OPTIONS);
+ configCredentialsToAzClient(blobServiceClientBuilder, config);
+ return blobServiceClientBuilder.buildClient();
}
- private static void configCredentialsToAzClient(BlobContainerClientBuilder
builder,
+ private static void configCredentialsToAzClient(BlobServiceClientBuilder
builder,
AzBlobStorageClientConfig config) {
if (config.isAnonymousAuth()) {
StorageSharedKeyCredential creds =
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AbstractCloudExternalFileWriterFactory.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AbstractCloudExternalFileWriterFactory.java
index 198b3ad81b..7c8ea03e5f 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AbstractCloudExternalFileWriterFactory.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AbstractCloudExternalFileWriterFactory.java
@@ -63,6 +63,10 @@ abstract class AbstractCloudExternalFileWriterFactory
implements IExternalFileWr
abstract ICloudClient createCloudClient(IApplicationContext appCtx) throws
CompilationException;
+ abstract String getAdapterName();
+
+ abstract int getPathMaxLengthInBytes();
+
abstract boolean isNoContainerFoundException(IOException e);
abstract boolean isSdkException(Throwable e);
@@ -110,8 +114,7 @@ abstract class AbstractCloudExternalFileWriterFactory
implements IExternalFileWr
if (staticPath != null) {
if (isExceedingMaxLength(staticPath,
S3ExternalFileWriter.MAX_LENGTH_IN_BYTES)) {
throw new
CompilationException(ErrorCode.WRITE_PATH_LENGTH_EXCEEDS_MAX_LENGTH,
pathSourceLocation,
- staticPath, S3ExternalFileWriter.MAX_LENGTH_IN_BYTES,
- ExternalDataConstants.KEY_ADAPTER_NAME_AWS_S3);
+ staticPath, getPathMaxLengthInBytes(),
getAdapterName());
}
if (!testClient.isEmptyPrefix(bucket, staticPath)) {
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AzureExternalFileWriter.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AzureExternalFileWriter.java
new file mode 100644
index 0000000000..f146ad7ea8
--- /dev/null
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AzureExternalFileWriter.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.cloud.writer;
+
+import org.apache.asterix.cloud.clients.ICloudClient;
+import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.asterix.external.util.azure.blob_storage.AzureConstants;
+import org.apache.asterix.runtime.writer.IExternalPrinter;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
+import org.apache.hyracks.api.exceptions.SourceLocation;
+
+import com.azure.core.exception.AzureException;
+
+final class AzureExternalFileWriter extends AbstractCloudExternalFileWriter {
+
+ AzureExternalFileWriter(IExternalPrinter printer, ICloudClient
cloudClient, String bucket, boolean partitionedPath,
+ IWarningCollector warningCollector, SourceLocation
pathSourceLocation) {
+ super(printer, cloudClient, bucket, partitionedPath, warningCollector,
pathSourceLocation);
+ }
+
+ @Override
+ String getAdapterName() {
+ return ExternalDataConstants.KEY_ADAPTER_NAME_AZURE_BLOB;
+ }
+
+ @Override
+ int getPathMaxLengthInBytes() {
+ return AzureConstants.MAX_KEY_LENGTH_IN_BYTES;
+ }
+
+ @Override
+ boolean isSdkException(Exception e) {
+ return e instanceof AzureException;
+ }
+}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AzureExternalFileWriterFactory.java
similarity index 73%
copy from
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java
copy to
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AzureExternalFileWriterFactory.java
index d268efd212..f876a5c1ac 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AzureExternalFileWriterFactory.java
@@ -22,12 +22,13 @@ import java.io.IOException;
import org.apache.asterix.cloud.clients.ICloudClient;
import org.apache.asterix.cloud.clients.ICloudGuardian;
-import org.apache.asterix.cloud.clients.aws.s3.S3ClientConfig;
-import org.apache.asterix.cloud.clients.aws.s3.S3CloudClient;
+import
org.apache.asterix.cloud.clients.azure.blobstorage.AzBlobStorageClientConfig;
+import
org.apache.asterix.cloud.clients.azure.blobstorage.AzBlobStorageCloudClient;
import org.apache.asterix.common.api.IApplicationContext;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.external.util.ExternalDataConstants;
-import org.apache.asterix.external.util.aws.s3.S3AuthUtils;
+import org.apache.asterix.external.util.azure.blob_storage.AzureConstants;
+import org.apache.asterix.external.util.azure.blob_storage.AzureUtils;
import org.apache.asterix.runtime.writer.ExternalFileWriterConfiguration;
import org.apache.asterix.runtime.writer.IExternalFileWriter;
import org.apache.asterix.runtime.writer.IExternalFileWriterFactory;
@@ -38,16 +39,17 @@ import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.IWarningCollector;
-import software.amazon.awssdk.core.exception.SdkException;
+import com.azure.core.exception.AzureException;
+
import software.amazon.awssdk.services.s3.model.NoSuchBucketException;
-public final class S3ExternalFileWriterFactory extends
AbstractCloudExternalFileWriterFactory {
+public final class AzureExternalFileWriterFactory extends
AbstractCloudExternalFileWriterFactory {
private static final long serialVersionUID = 4551318140901866805L;
static final char SEPARATOR = '/';
public static final IExternalFileWriterFactoryProvider PROVIDER = new
IExternalFileWriterFactoryProvider() {
@Override
public IExternalFileWriterFactory
create(ExternalFileWriterConfiguration configuration) {
- return new S3ExternalFileWriterFactory(configuration);
+ return new AzureExternalFileWriterFactory(configuration);
}
@Override
@@ -56,18 +58,28 @@ public final class S3ExternalFileWriterFactory extends
AbstractCloudExternalFile
}
};
- private S3ExternalFileWriterFactory(ExternalFileWriterConfiguration
externalConfig) {
+ private AzureExternalFileWriterFactory(ExternalFileWriterConfiguration
externalConfig) {
super(externalConfig);
cloudClient = null;
}
@Override
ICloudClient createCloudClient(IApplicationContext appCtx) throws
CompilationException {
- S3ClientConfig config = S3ClientConfig.of(configuration,
writeBufferSize);
- return new S3CloudClient(config, S3AuthUtils.buildAwsS3Client(appCtx,
configuration),
+ AzBlobStorageClientConfig config =
AzBlobStorageClientConfig.of(configuration, writeBufferSize);
+ return new AzBlobStorageCloudClient(config,
AzureUtils.buildAzureBlobClient(appCtx, configuration),
ICloudGuardian.NoOpCloudGuardian.INSTANCE);
}
+ @Override
+ String getAdapterName() {
+ return ExternalDataConstants.KEY_ADAPTER_NAME_AZURE_BLOB;
+ }
+
+ @Override
+ int getPathMaxLengthInBytes() {
+ return AzureConstants.MAX_KEY_LENGTH_IN_BYTES;
+ }
+
@Override
boolean isNoContainerFoundException(IOException e) {
return e.getCause() instanceof NoSuchBucketException;
@@ -75,7 +87,7 @@ public final class S3ExternalFileWriterFactory extends
AbstractCloudExternalFile
@Override
boolean isSdkException(Throwable e) {
- return e instanceof SdkException;
+ return e instanceof AzureException;
}
@Override
@@ -85,7 +97,7 @@ public final class S3ExternalFileWriterFactory extends
AbstractCloudExternalFile
String bucket =
configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
IExternalPrinter printer = printerFactory.createPrinter();
IWarningCollector warningCollector = context.getWarningCollector();
- return new S3ExternalFileWriter(printer, cloudClient, bucket,
staticPath == null, warningCollector,
+ return new AzureExternalFileWriter(printer, cloudClient, bucket,
staticPath == null, warningCollector,
pathSourceLocation);
}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/GCSExternalFileWriterFactory.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/GCSExternalFileWriterFactory.java
index 63a8366c3c..c07737395d 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/GCSExternalFileWriterFactory.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/GCSExternalFileWriterFactory.java
@@ -27,6 +27,7 @@ import
org.apache.asterix.cloud.clients.google.gcs.GCSCloudClient;
import org.apache.asterix.common.api.IApplicationContext;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.asterix.external.util.google.gcs.GCSConstants;
import org.apache.asterix.external.util.google.gcs.GCSUtils;
import org.apache.asterix.runtime.writer.ExternalFileWriterConfiguration;
import org.apache.asterix.runtime.writer.IExternalFileWriter;
@@ -68,6 +69,16 @@ public final class GCSExternalFileWriterFactory extends
AbstractCloudExternalFil
ICloudGuardian.NoOpCloudGuardian.INSTANCE);
}
+ @Override
+ String getAdapterName() {
+ return ExternalDataConstants.KEY_ADAPTER_NAME_GCS;
+ }
+
+ @Override
+ int getPathMaxLengthInBytes() {
+ return GCSConstants.MAX_KEY_LENGTH_IN_BYTES;
+ }
+
@Override
boolean isNoContainerFoundException(IOException e) {
return e.getCause() instanceof StorageException;
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java
index d268efd212..a95b9f260e 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java
@@ -28,6 +28,7 @@ import org.apache.asterix.common.api.IApplicationContext;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.aws.s3.S3AuthUtils;
+import org.apache.asterix.external.util.aws.s3.S3Constants;
import org.apache.asterix.runtime.writer.ExternalFileWriterConfiguration;
import org.apache.asterix.runtime.writer.IExternalFileWriter;
import org.apache.asterix.runtime.writer.IExternalFileWriterFactory;
@@ -68,6 +69,16 @@ public final class S3ExternalFileWriterFactory extends
AbstractCloudExternalFile
ICloudGuardian.NoOpCloudGuardian.INSTANCE);
}
+ @Override
+ String getAdapterName() {
+ return ExternalDataConstants.KEY_ADAPTER_NAME_AWS_S3;
+ }
+
+ @Override
+ int getPathMaxLengthInBytes() {
+ return S3Constants.MAX_KEY_LENGTH_IN_BYTES;
+ }
+
@Override
boolean isNoContainerFoundException(IOException e) {
return e.getCause() instanceof NoSuchBucketException;
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index 9e9df8be73..1cd9d0b884 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -396,7 +396,8 @@ public class ExternalDataConstants {
static {
WRITER_SUPPORTED_FORMATS = Set.of(FORMAT_JSON_LOWER_CASE,
FORMAT_PARQUET, FORMAT_CSV_LOWER_CASE);
WRITER_SUPPORTED_ADAPTERS =
Set.of(ALIAS_LOCALFS_ADAPTER.toLowerCase(),
KEY_ADAPTER_NAME_AWS_S3.toLowerCase(),
- KEY_ADAPTER_NAME_GCS.toLowerCase(),
KEY_ADAPTER_NAME_HDFS.toLowerCase());
+ KEY_ADAPTER_NAME_GCS.toLowerCase(),
KEY_ADAPTER_NAME_HDFS.toLowerCase(),
+ KEY_ADAPTER_NAME_AZURE_BLOB.toLowerCase());
TEXTUAL_WRITER_SUPPORTED_COMPRESSION = Set.of(KEY_COMPRESSION_GZIP);
PARQUET_WRITER_SUPPORTED_COMPRESSION =
Set.of(KEY_COMPRESSION_GZIP, KEY_COMPRESSION_SNAPPY,
KEY_COMPRESSION_ZSTD);
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
index a67cd64f5f..807e887bf3 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
@@ -23,6 +23,9 @@ public class S3Constants {
throw new AssertionError("do not instantiate");
}
+ // Key max length
+ public static final int MAX_KEY_LENGTH_IN_BYTES = 1024;
+
// Authentication specific parameters
public static final String REGION_FIELD_NAME = "region";
public static final String CROSS_REGION_FIELD_NAME = "crossRegion";
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob_storage/AzureConstants.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob_storage/AzureConstants.java
index 01ee148574..d8ddfb4e7d 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob_storage/AzureConstants.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob_storage/AzureConstants.java
@@ -30,6 +30,9 @@ public class AzureConstants {
throw new AssertionError("do not instantiate");
}
+ // Key max length
+ public static final int MAX_KEY_LENGTH_IN_BYTES = 1024;
+
public static final HttpLogOptions HTTP_LOG_OPTIONS = new HttpLogOptions();
static {
HTTP_LOG_OPTIONS.setLogLevel(HttpLogDetailLevel.BASIC);
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSConstants.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSConstants.java
index 2613f34e8f..bdfe563b23 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSConstants.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSConstants.java
@@ -23,6 +23,9 @@ public class GCSConstants {
throw new AssertionError("do not instantiate");
}
+ // Key max length
+ public static final int MAX_KEY_LENGTH_IN_BYTES = 1024;
+
public static final String APPLICATION_DEFAULT_CREDENTIALS_FIELD_NAME =
"applicationDefaultCredentials";
public static final String JSON_CREDENTIALS_FIELD_NAME = "jsonCredentials";
public static final String ENDPOINT_FIELD_NAME = "endpoint";
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java
index e88b1deb1e..0f803c65ce 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java
@@ -23,6 +23,7 @@ import java.util.Map;
import java.util.zip.Deflater;
import org.apache.asterix.cloud.parquet.ParquetSinkExternalWriterFactory;
+import org.apache.asterix.cloud.writer.AzureExternalFileWriterFactory;
import org.apache.asterix.cloud.writer.GCSExternalFileWriterFactory;
import org.apache.asterix.cloud.writer.S3ExternalFileWriterFactory;
import org.apache.asterix.common.dataflow.ICcApplicationContext;
@@ -79,6 +80,7 @@ public class ExternalWriterProvider {
addCreator(ExternalDataConstants.KEY_ADAPTER_NAME_AWS_S3,
S3ExternalFileWriterFactory.PROVIDER);
addCreator(ExternalDataConstants.KEY_ADAPTER_NAME_GCS,
GCSExternalFileWriterFactory.PROVIDER);
addCreator(ExternalDataConstants.KEY_ADAPTER_NAME_HDFS,
HDFSExternalFileWriterFactory.PROVIDER);
+ addCreator(ExternalDataConstants.KEY_ADAPTER_NAME_AZURE_BLOB,
AzureExternalFileWriterFactory.PROVIDER);
}
private static IExternalFileWriterFactory
createWriterFactory(ICcApplicationContext appCtx, IWriteDataSink sink,
