This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 7e7960c090942a06046b04163c9a186463998c1c Author: ayush.tripathi <[email protected]> AuthorDate: Fri Oct 11 15:35:20 2024 +0530 [ASTERIXDB-3503][EXT] Deltalake invalid type error code change - user model changes: no - storage format changes: no - interface changes: no Ext-ref: MB-63840 Change-Id: Ic0c12e32b85987cbc14d28e83fb2be644b922590 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18952 Integration-Tests: Jenkins <[email protected]> Reviewed-by: Ayush Tripathi <[email protected]> Reviewed-by: Murtadha Hubail <[email protected]> Tested-by: Murtadha Hubail <[email protected]> Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19008 Tested-by: Jenkins <[email protected]> --- .../common/deltalake-empty/deltalake-empty.00.ddl.sqlpp | 2 +- .../deltalake-invalid-file-format.00.ddl.sqlpp} | 7 ++++--- .../deltalake-modified-data/deltalake-modified-data.00.ddl.sqlpp | 2 +- .../deltalake-multiple_file_read.00.ddl.sqlpp | 2 +- .../src/test/resources/runtimets/testsuite_external_dataset_s3.xml | 7 +++++++ .../main/java/org/apache/asterix/common/exceptions/ErrorCode.java | 2 +- .../asterix-common/src/main/resources/asx_errormsg/en.properties | 2 +- .../java/org/apache/asterix/external/util/ExternalDataUtils.java | 5 +++-- 8 files changed, 19 insertions(+), 10 deletions(-) diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-empty/deltalake-empty.00.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-empty/deltalake-empty.00.ddl.sqlpp index f21f0131fc..2e77bebfdc 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-empty/deltalake-empty.00.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-empty/deltalake-empty.00.ddl.sqlpp @@ -26,7 +26,7 @@ CREATE TYPE DeltalakeTableType as { }; - CREATE EXTERNAL DATASET DeltalakeDataset(DeltalakeTableType) USING %adapter% + CREATE EXTERNAL COLLECTION DeltalakeDataset(DeltalakeTableType) USING %adapter% ( %template%, ("container"="playground"), diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-modified-data/deltalake-modified-data.00.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-invalid-file-format/deltalake-invalid-file-format.00.ddl.sqlpp similarity index 83% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-modified-data/deltalake-modified-data.00.ddl.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-invalid-file-format/deltalake-invalid-file-format.00.ddl.sqlpp index 8adab7721c..98799e1be8 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-modified-data/deltalake-modified-data.00.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-invalid-file-format/deltalake-invalid-file-format.00.ddl.sqlpp @@ -26,10 +26,11 @@ DROP DATAVERSE test IF EXISTS; CREATE TYPE DeltalakeTableType as { }; - CREATE EXTERNAL DATASET DeltalakeDataset(DeltalakeTableType) USING %adapter% + CREATE EXTERNAL COLLECTION DeltalakeDataset(DeltalakeTableType) USING %adapter% ( %template%, ("container"="playground"), - ("definition"="delta-data/modified_delta_table"), - ("table-format" = "delta") + ("definition"="delta-data/my-table-empty"), + ("table-format" = "delta"), + ("format" = "avro") ); \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-modified-data/deltalake-modified-data.00.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-modified-data/deltalake-modified-data.00.ddl.sqlpp index 8adab7721c..9238da6d8d 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-modified-data/deltalake-modified-data.00.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-modified-data/deltalake-modified-data.00.ddl.sqlpp @@ -26,7 +26,7 @@ DROP DATAVERSE test IF EXISTS; CREATE TYPE DeltalakeTableType as { }; - CREATE EXTERNAL DATASET DeltalakeDataset(DeltalakeTableType) USING %adapter% + CREATE EXTERNAL COLLECTION DeltalakeDataset(DeltalakeTableType) USING %adapter% ( %template%, ("container"="playground"), diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-multiple-file-read/deltalake-multiple_file_read.00.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-multiple-file-read/deltalake-multiple_file_read.00.ddl.sqlpp index 61c5c3ac38..9aa0380e81 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-multiple-file-read/deltalake-multiple_file_read.00.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-multiple-file-read/deltalake-multiple_file_read.00.ddl.sqlpp @@ -26,7 +26,7 @@ DROP DATAVERSE test IF EXISTS; CREATE TYPE DeltalakeTableType as { }; - CREATE EXTERNAL DATASET DeltalakeDataset(DeltalakeTableType) USING %adapter% + CREATE EXTERNAL COLLECTION DeltalakeDataset(DeltalakeTableType) USING %adapter% ( %template%, ("container"="playground"), diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml index 4e94791e2d..2c7af49eb0 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml @@ -320,6 +320,13 @@ <output-dir compare="Text">common/deltalake-modified-data</output-dir> </compilation-unit> </test-case> + <test-case FilePath="external-dataset"> + <compilation-unit name="common/deltalake-invalid-file-format"> + <placeholder name="adapter" value="S3" /> + <output-dir compare="Text">none</output-dir> + <expected-error>Supported file format for 'delta' tables is 'parquet', but 'avro' was provided.</expected-error> + </compilation-unit> + </test-case> <test-case FilePath="external-dataset"> <compilation-unit name="common/avro/avro-types/avro-map"> <placeholder name="adapter" value="S3" /> diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java index 985a4f699d..091fb6774f 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java @@ -97,7 +97,6 @@ public enum ErrorCode implements IError { UNSUPPORTED_COLUMN_TYPE(67), INVALID_KEY_TYPE(68), FAILED_TO_READ_KEY(69), - INVALID_DELTA_PARAMETER(70), UNSUPPORTED_JRE(100), @@ -304,6 +303,7 @@ public enum ErrorCode implements IError { DUPLICATE_FIELD_IN_PRIMARY_KEY(1198), INCOMPATIBLE_FIELDS_IN_PRIMARY_KEY(1199), PREFIX_SHOULD_NOT_START_WITH_SLASH(1200), + INVALID_DELTA_TABLE_FORMAT(1201), // Feed errors DATAFLOW_ILLEGAL_STATE(3001), diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties index b5c06e3aa2..8e3ccffff3 100644 --- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties +++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties @@ -41,7 +41,6 @@ 1009 = A returning expression cannot contain dataset access 37,1091 = Type mismatch: expected value of type %1$s, but got the value of type %2$s 51 = Incomparable input types: %1$s and %2$s -70 = Table Type 'delta' supports parquet file formats only. # Data errors 6 = Invalid format for %1$s in %2$s @@ -306,6 +305,7 @@ 1198 = Duplicate field '%1$s' in primary key 1199 = Fields '%1$s' and '%2$s' are incompatible for primary key 1200 = Prefix should not start with "/". Prefix: '%1$s' +1201 = Supported file format for 'delta' tables is 'parquet', but '%1$s' was provided. # Feed Errors 3001 = Illegal state. diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java index 8d332610e8..b4314b99f0 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java @@ -473,7 +473,7 @@ public class ExternalDataUtils { } if (configuration.containsKey(ExternalDataConstants.TABLE_FORMAT)) { - if (configuration.get(ExternalDataConstants.TABLE_FORMAT).equals(ExternalDataConstants.FORMAT_DELTA)) { + if (isDeltaTable(configuration)) { configuration.put(ExternalDataConstants.KEY_PARSER, ExternalDataConstants.FORMAT_NOOP); configuration.put(ExternalDataConstants.KEY_FORMAT, ExternalDataConstants.FORMAT_PARQUET); } @@ -494,7 +494,8 @@ public class ExternalDataUtils { public static void validateDeltaTableProperties(Map<String, String> configuration) throws CompilationException { if (!(configuration.get(ExternalDataConstants.KEY_FORMAT) == null || configuration.get(ExternalDataConstants.KEY_FORMAT).equals(ExternalDataConstants.FORMAT_PARQUET))) { - throw new CompilationException(ErrorCode.INVALID_DELTA_PARAMETER); + throw new CompilationException(ErrorCode.INVALID_DELTA_TABLE_FORMAT, + configuration.get(ExternalDataConstants.KEY_FORMAT)); } }
