>From Wail Alkowaileet <[email protected]>: Wail Alkowaileet has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18778 )
Change subject: [ASTERIXDB-3392] Fix format in tests ...................................................................... [ASTERIXDB-3392] Fix format in tests Change-Id: I480866ba09e519f9d1242fdfdc9e2a70f7578f13 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18778 Reviewed-by: Wail Alkowaileet <[email protected]> Tested-by: Wail Alkowaileet <[email protected]> Integration-Tests: Wail Alkowaileet <[email protected]> --- M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.15.update.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.11.update.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.05.update.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.04.update.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.12.update.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.06.update.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.14.update.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.13.update.sqlpp M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/WriterValidationUtil.java M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.03.update.sqlpp M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/parquet/AsterixParquetTypeMap.java M asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties 15 files changed, 37 insertions(+), 75 deletions(-) Approvals: Wail Alkowaileet: Looks good to me, approved; Verified; Verified diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.03.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.03.update.sqlpp index 5fb0a9b..ce68cbd 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.03.update.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.03.update.sqlpp @@ -27,15 +27,7 @@ ) toWriter TO S3 PATH ("copy-to-result", "parquet-error-checks3") -TYPE ( { - id : string, - name : string, - nested : - { - first : string, - second : string - } - } +TYPE ( {id : string, name : string, nested: { first : string, second : string} } ) WITH { "accessKeyId":"dummyAccessKey", diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.04.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.04.update.sqlpp index 8b1efc8..5e8d730 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.04.update.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.04.update.sqlpp @@ -26,14 +26,7 @@ ) toWriter TO S3 PATH ("copy-to-result", "parquet-error-checks4") -TYPE ( { - id : bigint, - name : string, - nested: - { - first : string - } - } ) +TYPE ( {id : bigint,name : string, nested: { first : string } } ) WITH { "accessKeyId":"dummyAccessKey", "secretAccessKey":"dummySecretKey", diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.05.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.05.update.sqlpp index 79b2d1a..cda2519 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.05.update.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.05.update.sqlpp @@ -25,11 +25,7 @@ ) toWriter TO S3 PATH ("copy-to-result", "parquet-error-checks5") -TYPE ( { - id : bigint, - name : string, - nested : string - } ) +TYPE ( { id : bigint, name : string, nested : string } ) WITH { "accessKeyId":"dummyAccessKey", "secretAccessKey":"dummySecretKey", diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.06.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.06.update.sqlpp index 3e6ac48..a6db11d 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.06.update.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.06.update.sqlpp @@ -25,16 +25,7 @@ ) toWriter TO S3 PATH ("copy-to-result", "parquet-error-checks6") -TYPE ( { - id : bigint, - name : { - first : string - }, - nested:{ - first : string, - second : string - } - } ) +TYPE ( {id : bigint,name : { first : string },nested:{first : string,second : string} } ) WITH { "accessKeyId":"dummyAccessKey", "secretAccessKey":"dummySecretKey", diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.11.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.11.update.sqlpp index f22071a..ca2cfc1 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.11.update.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.11.update.sqlpp @@ -24,9 +24,7 @@ ) toWriter TO S3 PATH ("copy-to-result", "parquet-error-checks11") -TYPE ( { - id : int , name : binary - } ) +TYPE ( { id : int , name : binary } ) WITH { "accessKeyId":"dummyAccessKey", "secretAccessKey":"dummySecretKey", diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.12.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.12.update.sqlpp index d5d11eb..86e55e5 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.12.update.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.12.update.sqlpp @@ -24,10 +24,7 @@ ) toWriter TO S3 PATH ("copy-to-result", "parquet-error-checks12") -TYPE ( { - id : int, - name : string - } ) +TYPE ( { id : int, name : string } ) WITH { "accessKeyId":"dummyAccessKey", "secretAccessKey":"dummySecretKey", diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.13.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.13.update.sqlpp index 75245f1..281610b 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.13.update.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.13.update.sqlpp @@ -24,11 +24,7 @@ ) toWriter TO S3 PATH ("copy-to-result", "parquet-error-checks13") -TYPE ( { - id : int, - name : string, - list : [int,string] - } ) +TYPE ( { id : int, name : string, list : [int,string]} ) WITH { "accessKeyId":"dummyAccessKey", "secretAccessKey":"dummySecretKey", diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.14.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.14.update.sqlpp index 0becb36..1d1c6af 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.14.update.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.14.update.sqlpp @@ -24,11 +24,7 @@ ) toWriter TO S3 PATH ("copy-to-result", "parquet-error-checks14") -TYPE ( { - id : int, - name : string, - list : [int | - } ) +TYPE ( {id : int, name : string, list : [int |} ) WITH { "accessKeyId":"dummyAccessKey", "secretAccessKey":"dummySecretKey", diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.15.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.15.update.sqlpp index cc67f79..a26624a 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.15.update.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.15.update.sqlpp @@ -24,11 +24,7 @@ ) toWriter TO S3 PATH ("copy-to-result", "parquet-error-checks15") -TYPE ( { - id : int, - name : string, - list : [int] ) - +TYPE ( {id : int, name : string, list : [int] ) WITH { "accessKeyId":"dummyAccessKey", "secretAccessKey":"dummySecretKey", diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml index 1107dda..d02d551 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml @@ -117,7 +117,7 @@ <expected-error>ASX1202: Unsupported compression scheme rar. Supported schemes for parquet are [gzip, snappy, zstd]</expected-error> <expected-error>ASX1001: Syntax error</expected-error> <expected-error>ASX1204: 'binary' type not supported in parquet format</expected-error> - <expected-error>ASX1205: Invalid Parquet Writer Version provided.Supported values: 1,2</expected-error> + <expected-error>ASX1205: Invalid Parquet Writer Version provided '3'. Supported values: [1, 2]</expected-error> <expected-error>ASX1001: Syntax error</expected-error> <expected-error>ASX1001: Syntax error</expected-error> <expected-error>ASX1001: Syntax error</expected-error> diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties index 716dcf6..748a2b9 100644 --- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties +++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties @@ -309,7 +309,7 @@ 1202 = Unsupported compression scheme %1$s. Supported schemes for %2$s are %3$s 1203 = Invalid schema provided: '%1$s' 1204 = '%1$s' type not supported in parquet format -1205 = Invalid Parquet Writer Version provided.Supported values: 1,2 +1205 = Invalid Parquet Writer Version provided '%1$s'. Supported values: %2$s # Feed Errors 3001 = Illegal state. diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java index 02c2070..0407005 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java @@ -336,6 +336,7 @@ public static final Set<String> WRITER_SUPPORTED_ADAPTERS; public static final Set<String> TEXTUAL_WRITER_SUPPORTED_COMPRESSION; public static final Set<String> PARQUET_WRITER_SUPPORTED_COMPRESSION; + public static final Set<String> PARQUET_WRITER_SUPPORTED_VERSION; public static final int PARQUET_DICTIONARY_PAGE_SIZE = 1048576; static { @@ -345,6 +346,7 @@ TEXTUAL_WRITER_SUPPORTED_COMPRESSION = Set.of(KEY_COMPRESSION_GZIP); PARQUET_WRITER_SUPPORTED_COMPRESSION = Set.of(KEY_COMPRESSION_GZIP, KEY_COMPRESSION_SNAPPY, KEY_COMPRESSION_ZSTD); + PARQUET_WRITER_SUPPORTED_VERSION = Set.of(PARQUET_WRITER_VERSION_VALUE_1, PARQUET_WRITER_VERSION_VALUE_2); } public static class ParquetOptions { diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/WriterValidationUtil.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/WriterValidationUtil.java index 5c3585b..5059ec8 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/WriterValidationUtil.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/WriterValidationUtil.java @@ -27,8 +27,6 @@ import static org.apache.asterix.external.util.ExternalDataConstants.KEY_PARQUET_ROW_GROUP_SIZE; import static org.apache.asterix.external.util.ExternalDataConstants.KEY_WRITER_MAX_RESULT; import static org.apache.asterix.external.util.ExternalDataConstants.PARQUET_WRITER_VERSION_KEY; -import static org.apache.asterix.external.util.ExternalDataConstants.PARQUET_WRITER_VERSION_VALUE_1; -import static org.apache.asterix.external.util.ExternalDataConstants.PARQUET_WRITER_VERSION_VALUE_2; import static org.apache.asterix.external.util.ExternalDataConstants.WRITER_MAX_RESULT_MINIMUM; import java.util.List; @@ -79,18 +77,14 @@ validateParquetCompression(configuration, sourceLocation); validateParquetRowGroupSize(configuration); validateParquetPageSize(configuration); - validateVersion(configuration); + validateVersion(configuration, sourceLocation); } - private static void validateVersion(Map<String, String> configuration) throws CompilationException { + private static void validateVersion(Map<String, String> configuration, SourceLocation sourceLocation) + throws CompilationException { String version = configuration.get(PARQUET_WRITER_VERSION_KEY); - if (version == null) { - return; - } - if (version.equals(PARQUET_WRITER_VERSION_VALUE_1) || version.equals(PARQUET_WRITER_VERSION_VALUE_2)) { - return; - } - throw CompilationException.create(ErrorCode.INVALID_PARQUET_WRITER_VERSION); + checkSupported(PARQUET_WRITER_VERSION_KEY, version, ExternalDataConstants.PARQUET_WRITER_SUPPORTED_VERSION, + ErrorCode.INVALID_PARQUET_WRITER_VERSION, sourceLocation, true); } private static void validateParquetRowGroupSize(Map<String, String> configuration) throws CompilationException { diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/parquet/AsterixParquetTypeMap.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/parquet/AsterixParquetTypeMap.java index 410c951..0dcdb3a 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/parquet/AsterixParquetTypeMap.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/parquet/AsterixParquetTypeMap.java @@ -31,7 +31,7 @@ Map.entry(ATypeTag.STRING, PrimitiveType.PrimitiveTypeName.BINARY), Map.entry(ATypeTag.TINYINT, PrimitiveType.PrimitiveTypeName.INT32), Map.entry(ATypeTag.SMALLINT, PrimitiveType.PrimitiveTypeName.INT32), - Map.entry(ATypeTag.INTEGER, PrimitiveType.PrimitiveTypeName.INT64), + Map.entry(ATypeTag.INTEGER, PrimitiveType.PrimitiveTypeName.INT32), Map.entry(ATypeTag.BIGINT, PrimitiveType.PrimitiveTypeName.INT64), Map.entry(ATypeTag.FLOAT, PrimitiveType.PrimitiveTypeName.FLOAT), Map.entry(ATypeTag.DOUBLE, PrimitiveType.PrimitiveTypeName.DOUBLE), diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java index 23b9f93..ee7b3fc 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java @@ -157,14 +157,12 @@ } private static ParquetProperties.WriterVersion getParquetWriterVersion(Map<String, String> configuration) { - - if (configuration.get(ExternalDataConstants.PARQUET_WRITER_VERSION_KEY) == null) { - return ParquetProperties.WriterVersion.PARQUET_1_0; - } else if (configuration.get(ExternalDataConstants.PARQUET_WRITER_VERSION_KEY) - .equals(ExternalDataConstants.PARQUET_WRITER_VERSION_VALUE_2)) { + String writerVersionString = configuration.getOrDefault(ExternalDataConstants.PARQUET_WRITER_VERSION_KEY, + ExternalDataConstants.PARQUET_WRITER_VERSION_VALUE_1); + if (writerVersionString.equals(ExternalDataConstants.PARQUET_WRITER_VERSION_VALUE_2)) { return ParquetProperties.WriterVersion.PARQUET_2_0; - } else - return ParquetProperties.WriterVersion.PARQUET_1_0; + } + return ParquetProperties.WriterVersion.PARQUET_1_0; } private static String getRowGroupSize(Map<String, String> configuration) { -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18778 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I480866ba09e519f9d1242fdfdc9e2a70f7578f13 Gerrit-Change-Number: 18778 Gerrit-PatchSet: 4 Gerrit-Owner: [email protected] Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Wail Alkowaileet <[email protected]> Gerrit-Reviewer: [email protected] Gerrit-MessageType: merged
