This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 9a449913bb [ASTERIXDB-3503][EXT] Deltalake invalid type error code
change
9a449913bb is described below
commit 9a449913bb79ed5d02f36b9dc59db46449ff9ce4
Author: ayush.tripathi <[email protected]>
AuthorDate: Fri Oct 11 15:35:20 2024 +0530
[ASTERIXDB-3503][EXT] Deltalake invalid type error code change
- user model changes: no
- storage format changes: no
- interface changes: no
Change-Id: Ic0c12e32b85987cbc14d28e83fb2be644b922590
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18952
Integration-Tests: Jenkins <[email protected]>
Reviewed-by: Ayush Tripathi <[email protected]>
Reviewed-by: Murtadha Hubail <[email protected]>
Tested-by: Murtadha Hubail <[email protected]>
---
.../common/deltalake-empty/deltalake-empty.00.ddl.sqlpp | 2 +-
.../deltalake-invalid-file-format.00.ddl.sqlpp} | 7 ++++---
.../deltalake-modified-data/deltalake-modified-data.00.ddl.sqlpp | 2 +-
.../deltalake-multiple_file_read.00.ddl.sqlpp | 2 +-
.../src/test/resources/runtimets/testsuite_external_dataset_s3.xml | 7 +++++++
.../main/java/org/apache/asterix/common/exceptions/ErrorCode.java | 4 +---
.../asterix-common/src/main/resources/asx_errormsg/en.properties | 2 +-
.../java/org/apache/asterix/external/util/ExternalDataUtils.java | 5 +++--
8 files changed, 19 insertions(+), 12 deletions(-)
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-empty/deltalake-empty.00.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-empty/deltalake-empty.00.ddl.sqlpp
index f21f0131fc..2e77bebfdc 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-empty/deltalake-empty.00.ddl.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-empty/deltalake-empty.00.ddl.sqlpp
@@ -26,7 +26,7 @@
CREATE TYPE DeltalakeTableType as {
};
- CREATE EXTERNAL DATASET DeltalakeDataset(DeltalakeTableType) USING %adapter%
+ CREATE EXTERNAL COLLECTION DeltalakeDataset(DeltalakeTableType) USING
%adapter%
(
%template%,
("container"="playground"),
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-modified-data/deltalake-modified-data.00.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-invalid-file-format/deltalake-invalid-file-format.00.ddl.sqlpp
similarity index 83%
copy from
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-modified-data/deltalake-modified-data.00.ddl.sqlpp
copy to
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-invalid-file-format/deltalake-invalid-file-format.00.ddl.sqlpp
index 8adab7721c..98799e1be8 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-modified-data/deltalake-modified-data.00.ddl.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-invalid-file-format/deltalake-invalid-file-format.00.ddl.sqlpp
@@ -26,10 +26,11 @@ DROP DATAVERSE test IF EXISTS;
CREATE TYPE DeltalakeTableType as {
};
- CREATE EXTERNAL DATASET DeltalakeDataset(DeltalakeTableType) USING %adapter%
+ CREATE EXTERNAL COLLECTION DeltalakeDataset(DeltalakeTableType) USING
%adapter%
(
%template%,
("container"="playground"),
- ("definition"="delta-data/modified_delta_table"),
- ("table-format" = "delta")
+ ("definition"="delta-data/my-table-empty"),
+ ("table-format" = "delta"),
+ ("format" = "avro")
);
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-modified-data/deltalake-modified-data.00.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-modified-data/deltalake-modified-data.00.ddl.sqlpp
index 8adab7721c..9238da6d8d 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-modified-data/deltalake-modified-data.00.ddl.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-modified-data/deltalake-modified-data.00.ddl.sqlpp
@@ -26,7 +26,7 @@ DROP DATAVERSE test IF EXISTS;
CREATE TYPE DeltalakeTableType as {
};
- CREATE EXTERNAL DATASET DeltalakeDataset(DeltalakeTableType) USING %adapter%
+ CREATE EXTERNAL COLLECTION DeltalakeDataset(DeltalakeTableType) USING
%adapter%
(
%template%,
("container"="playground"),
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-multiple-file-read/deltalake-multiple_file_read.00.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-multiple-file-read/deltalake-multiple_file_read.00.ddl.sqlpp
index 61c5c3ac38..9aa0380e81 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-multiple-file-read/deltalake-multiple_file_read.00.ddl.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-multiple-file-read/deltalake-multiple_file_read.00.ddl.sqlpp
@@ -26,7 +26,7 @@ DROP DATAVERSE test IF EXISTS;
CREATE TYPE DeltalakeTableType as {
};
- CREATE EXTERNAL DATASET DeltalakeDataset(DeltalakeTableType) USING %adapter%
+ CREATE EXTERNAL COLLECTION DeltalakeDataset(DeltalakeTableType) USING
%adapter%
(
%template%,
("container"="playground"),
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index 48e102fb1f..949c0a3584 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -364,6 +364,13 @@
<output-dir compare="Text">common/deltalake-modified-data</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="common/deltalake-invalid-file-format">
+ <placeholder name="adapter" value="S3" />
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>Supported file format for 'delta' tables is 'parquet',
but 'avro' was provided.</expected-error>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="external-dataset">
<compilation-unit name="common/avro/avro-types/avro-map">
<placeholder name="adapter" value="S3" />
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index 25e24bb53d..7c89bc91e6 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -98,8 +98,6 @@ public enum ErrorCode implements IError {
UNSUPPORTED_COLUMN_TYPE(67),
INVALID_KEY_TYPE(68),
FAILED_TO_READ_KEY(69),
- INVALID_DELTA_PARAMETER(70),
-
UNSUPPORTED_JRE(100),
EXTERNAL_UDF_RESULT_TYPE_ERROR(200),
@@ -310,7 +308,7 @@ public enum ErrorCode implements IError {
INVALID_PARQUET_SCHEMA(1203),
TYPE_UNSUPPORTED_PARQUET_WRITE(1204),
INVALID_PARQUET_WRITER_VERSION(1205),
-
+ INVALID_DELTA_TABLE_FORMAT(1206),
// Feed errors
DATAFLOW_ILLEGAL_STATE(3001),
UTIL_DATAFLOW_UTILS_TUPLE_TOO_LARGE(3002),
diff --git
a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index 817127f539..d1e01f66f9 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -41,7 +41,6 @@
1009 = A returning expression cannot contain dataset access
37,1091 = Type mismatch: expected value of type %1$s, but got the value of
type %2$s
51 = Incomparable input types: %1$s and %2$s
-70 = Table Type 'delta' supports parquet file formats only.
# Data errors
6 = Invalid format for %1$s in %2$s
@@ -311,6 +310,7 @@
1203 = Invalid schema provided: '%1$s'
1204 = '%1$s' type not supported in parquet format
1205 = Invalid Parquet Writer Version provided '%1$s'. Supported values: %2$s
+1206 = Supported file format for 'delta' tables is 'parquet', but '%1$s' was
provided.
# Feed Errors
3001 = Illegal state.
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 8d332610e8..b4314b99f0 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -473,7 +473,7 @@ public class ExternalDataUtils {
}
if (configuration.containsKey(ExternalDataConstants.TABLE_FORMAT)) {
- if
(configuration.get(ExternalDataConstants.TABLE_FORMAT).equals(ExternalDataConstants.FORMAT_DELTA))
{
+ if (isDeltaTable(configuration)) {
configuration.put(ExternalDataConstants.KEY_PARSER,
ExternalDataConstants.FORMAT_NOOP);
configuration.put(ExternalDataConstants.KEY_FORMAT,
ExternalDataConstants.FORMAT_PARQUET);
}
@@ -494,7 +494,8 @@ public class ExternalDataUtils {
public static void validateDeltaTableProperties(Map<String, String>
configuration) throws CompilationException {
if (!(configuration.get(ExternalDataConstants.KEY_FORMAT) == null
||
configuration.get(ExternalDataConstants.KEY_FORMAT).equals(ExternalDataConstants.FORMAT_PARQUET)))
{
- throw new CompilationException(ErrorCode.INVALID_DELTA_PARAMETER);
+ throw new
CompilationException(ErrorCode.INVALID_DELTA_TABLE_FORMAT,
+ configuration.get(ExternalDataConstants.KEY_FORMAT));
}
}
