>From <[email protected]>:
[email protected] has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18778 )
Change subject: [ASTERIXDB-3392] Fix format in tests
......................................................................
[ASTERIXDB-3392] Fix format in tests
Change-Id: I480866ba09e519f9d1242fdfdc9e2a70f7578f13
---
M
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.15.update.sqlpp
M
asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
M
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java
M
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.11.update.sqlpp
M
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.05.update.sqlpp
M
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.04.update.sqlpp
M
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.12.update.sqlpp
M
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.06.update.sqlpp
M
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.14.update.sqlpp
M
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.13.update.sqlpp
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/WriterValidationUtil.java
M
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.03.update.sqlpp
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/parquet/AsterixParquetTypeMap.java
M asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
15 files changed, 34 insertions(+), 74 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/78/18778/1
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.03.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.03.update.sqlpp
index 5fb0a9b..ce68cbd 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.03.update.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.03.update.sqlpp
@@ -27,15 +27,7 @@
) toWriter
TO S3
PATH ("copy-to-result", "parquet-error-checks3")
-TYPE ( {
- id : string,
- name : string,
- nested :
- {
- first : string,
- second : string
- }
- }
+TYPE ( {id : string, name : string, nested: { first : string, second : string}
}
)
WITH {
"accessKeyId":"dummyAccessKey",
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.04.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.04.update.sqlpp
index 8b1efc8..5e8d730 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.04.update.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.04.update.sqlpp
@@ -26,14 +26,7 @@
) toWriter
TO S3
PATH ("copy-to-result", "parquet-error-checks4")
-TYPE ( {
- id : bigint,
- name : string,
- nested:
- {
- first : string
- }
- } )
+TYPE ( {id : bigint,name : string, nested: { first : string } } )
WITH {
"accessKeyId":"dummyAccessKey",
"secretAccessKey":"dummySecretKey",
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.05.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.05.update.sqlpp
index 79b2d1a..cda2519 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.05.update.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.05.update.sqlpp
@@ -25,11 +25,7 @@
) toWriter
TO S3
PATH ("copy-to-result", "parquet-error-checks5")
-TYPE ( {
- id : bigint,
- name : string,
- nested : string
- } )
+TYPE ( { id : bigint, name : string, nested : string } )
WITH {
"accessKeyId":"dummyAccessKey",
"secretAccessKey":"dummySecretKey",
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.06.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.06.update.sqlpp
index 3e6ac48..a6db11d 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.06.update.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.06.update.sqlpp
@@ -25,16 +25,7 @@
) toWriter
TO S3
PATH ("copy-to-result", "parquet-error-checks6")
-TYPE ( {
- id : bigint,
- name : {
- first : string
- },
- nested:{
- first : string,
- second : string
- }
- } )
+TYPE ( {id : bigint,name : { first : string },nested:{first : string,second :
string} } )
WITH {
"accessKeyId":"dummyAccessKey",
"secretAccessKey":"dummySecretKey",
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.11.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.11.update.sqlpp
index f22071a..ca2cfc1 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.11.update.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.11.update.sqlpp
@@ -24,9 +24,7 @@
) toWriter
TO S3
PATH ("copy-to-result", "parquet-error-checks11")
-TYPE ( {
- id : int , name : binary
- } )
+TYPE ( { id : int , name : binary } )
WITH {
"accessKeyId":"dummyAccessKey",
"secretAccessKey":"dummySecretKey",
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.12.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.12.update.sqlpp
index d5d11eb..86e55e5 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.12.update.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.12.update.sqlpp
@@ -24,10 +24,7 @@
) toWriter
TO S3
PATH ("copy-to-result", "parquet-error-checks12")
-TYPE ( {
- id : int,
- name : string
- } )
+TYPE ( { id : int, name : string } )
WITH {
"accessKeyId":"dummyAccessKey",
"secretAccessKey":"dummySecretKey",
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.13.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.13.update.sqlpp
index 75245f1..281610b 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.13.update.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.13.update.sqlpp
@@ -24,11 +24,7 @@
) toWriter
TO S3
PATH ("copy-to-result", "parquet-error-checks13")
-TYPE ( {
- id : int,
- name : string,
- list : [int,string]
- } )
+TYPE ( { id : int, name : string, list : [int,string]} )
WITH {
"accessKeyId":"dummyAccessKey",
"secretAccessKey":"dummySecretKey",
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.14.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.14.update.sqlpp
index 0becb36..1d1c6af 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.14.update.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.14.update.sqlpp
@@ -24,11 +24,7 @@
) toWriter
TO S3
PATH ("copy-to-result", "parquet-error-checks14")
-TYPE ( {
- id : int,
- name : string,
- list : [int |
- } )
+TYPE ( {id : int, name : string, list : [int |} )
WITH {
"accessKeyId":"dummyAccessKey",
"secretAccessKey":"dummySecretKey",
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.15.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.15.update.sqlpp
index cc67f79..a26624a 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.15.update.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.15.update.sqlpp
@@ -24,11 +24,7 @@
) toWriter
TO S3
PATH ("copy-to-result", "parquet-error-checks15")
-TYPE ( {
- id : int,
- name : string,
- list : [int] )
-
+TYPE ( {id : int, name : string, list : [int] )
WITH {
"accessKeyId":"dummyAccessKey",
"secretAccessKey":"dummySecretKey",
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index 1107dda..ac32e3d 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -117,7 +117,7 @@
<expected-error>ASX1202: Unsupported compression scheme rar. Supported
schemes for parquet are [gzip, snappy, zstd]</expected-error>
<expected-error>ASX1001: Syntax error</expected-error>
<expected-error>ASX1204: 'binary' type not supported in parquet
format</expected-error>
- <expected-error>ASX1205: Invalid Parquet Writer Version
provided.Supported values: 1,2</expected-error>
+ <expected-error>ASX1205: Invalid Parquet Writer Version provided '3'
.Supported values: [1, 2]</expected-error>
<expected-error>ASX1001: Syntax error</expected-error>
<expected-error>ASX1001: Syntax error</expected-error>
<expected-error>ASX1001: Syntax error</expected-error>
diff --git
a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index 716dcf6..b4bf830 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -309,7 +309,7 @@
1202 = Unsupported compression scheme %1$s. Supported schemes for %2$s are %3$s
1203 = Invalid schema provided: '%1$s'
1204 = '%1$s' type not supported in parquet format
-1205 = Invalid Parquet Writer Version provided.Supported values: 1,2
+1205 = Invalid Parquet Writer Version provided '%1$s' .Supported values: %2$s
# Feed Errors
3001 = Illegal state.
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index 02c2070..0407005 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -336,6 +336,7 @@
public static final Set<String> WRITER_SUPPORTED_ADAPTERS;
public static final Set<String> TEXTUAL_WRITER_SUPPORTED_COMPRESSION;
public static final Set<String> PARQUET_WRITER_SUPPORTED_COMPRESSION;
+ public static final Set<String> PARQUET_WRITER_SUPPORTED_VERSION;
public static final int PARQUET_DICTIONARY_PAGE_SIZE = 1048576;
static {
@@ -345,6 +346,7 @@
TEXTUAL_WRITER_SUPPORTED_COMPRESSION = Set.of(KEY_COMPRESSION_GZIP);
PARQUET_WRITER_SUPPORTED_COMPRESSION =
Set.of(KEY_COMPRESSION_GZIP, KEY_COMPRESSION_SNAPPY,
KEY_COMPRESSION_ZSTD);
+ PARQUET_WRITER_SUPPORTED_VERSION =
Set.of(PARQUET_WRITER_VERSION_VALUE_1, PARQUET_WRITER_VERSION_VALUE_2);
}
public static class ParquetOptions {
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/WriterValidationUtil.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/WriterValidationUtil.java
index 5c3585b..5059ec8 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/WriterValidationUtil.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/WriterValidationUtil.java
@@ -27,8 +27,6 @@
import static
org.apache.asterix.external.util.ExternalDataConstants.KEY_PARQUET_ROW_GROUP_SIZE;
import static
org.apache.asterix.external.util.ExternalDataConstants.KEY_WRITER_MAX_RESULT;
import static
org.apache.asterix.external.util.ExternalDataConstants.PARQUET_WRITER_VERSION_KEY;
-import static
org.apache.asterix.external.util.ExternalDataConstants.PARQUET_WRITER_VERSION_VALUE_1;
-import static
org.apache.asterix.external.util.ExternalDataConstants.PARQUET_WRITER_VERSION_VALUE_2;
import static
org.apache.asterix.external.util.ExternalDataConstants.WRITER_MAX_RESULT_MINIMUM;
import java.util.List;
@@ -79,18 +77,14 @@
validateParquetCompression(configuration, sourceLocation);
validateParquetRowGroupSize(configuration);
validateParquetPageSize(configuration);
- validateVersion(configuration);
+ validateVersion(configuration, sourceLocation);
}
- private static void validateVersion(Map<String, String> configuration)
throws CompilationException {
+ private static void validateVersion(Map<String, String> configuration,
SourceLocation sourceLocation)
+ throws CompilationException {
String version = configuration.get(PARQUET_WRITER_VERSION_KEY);
- if (version == null) {
- return;
- }
- if (version.equals(PARQUET_WRITER_VERSION_VALUE_1) ||
version.equals(PARQUET_WRITER_VERSION_VALUE_2)) {
- return;
- }
- throw
CompilationException.create(ErrorCode.INVALID_PARQUET_WRITER_VERSION);
+ checkSupported(PARQUET_WRITER_VERSION_KEY, version,
ExternalDataConstants.PARQUET_WRITER_SUPPORTED_VERSION,
+ ErrorCode.INVALID_PARQUET_WRITER_VERSION, sourceLocation,
true);
}
private static void validateParquetRowGroupSize(Map<String, String>
configuration) throws CompilationException {
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/parquet/AsterixParquetTypeMap.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/parquet/AsterixParquetTypeMap.java
index 410c951..0dcdb3a 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/parquet/AsterixParquetTypeMap.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/parquet/AsterixParquetTypeMap.java
@@ -31,7 +31,7 @@
Map.entry(ATypeTag.STRING,
PrimitiveType.PrimitiveTypeName.BINARY),
Map.entry(ATypeTag.TINYINT,
PrimitiveType.PrimitiveTypeName.INT32),
Map.entry(ATypeTag.SMALLINT,
PrimitiveType.PrimitiveTypeName.INT32),
- Map.entry(ATypeTag.INTEGER,
PrimitiveType.PrimitiveTypeName.INT64),
+ Map.entry(ATypeTag.INTEGER,
PrimitiveType.PrimitiveTypeName.INT32),
Map.entry(ATypeTag.BIGINT,
PrimitiveType.PrimitiveTypeName.INT64),
Map.entry(ATypeTag.FLOAT,
PrimitiveType.PrimitiveTypeName.FLOAT),
Map.entry(ATypeTag.DOUBLE,
PrimitiveType.PrimitiveTypeName.DOUBLE),
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java
index 23b9f93..17b8039 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java
@@ -157,14 +157,14 @@
}
private static ParquetProperties.WriterVersion
getParquetWriterVersion(Map<String, String> configuration) {
-
- if
(configuration.get(ExternalDataConstants.PARQUET_WRITER_VERSION_KEY) == null) {
+ String writerVersionString =
configuration.getOrDefault(ExternalDataConstants.PARQUET_WRITER_VERSION_KEY,
+ ExternalDataConstants.PARQUET_WRITER_VERSION_VALUE_1);
+ if
(writerVersionString.equals(ExternalDataConstants.PARQUET_WRITER_VERSION_VALUE_1))
{
return ParquetProperties.WriterVersion.PARQUET_1_0;
- } else if
(configuration.get(ExternalDataConstants.PARQUET_WRITER_VERSION_KEY)
- .equals(ExternalDataConstants.PARQUET_WRITER_VERSION_VALUE_2))
{
+ } else if
(writerVersionString.equals(ExternalDataConstants.PARQUET_WRITER_VERSION_VALUE_2))
{
return ParquetProperties.WriterVersion.PARQUET_2_0;
- } else
- return ParquetProperties.WriterVersion.PARQUET_1_0;
+ }
+ return ParquetProperties.WriterVersion.PARQUET_2_0;
}
private static String getRowGroupSize(Map<String, String> configuration) {
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18778
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I480866ba09e519f9d1242fdfdc9e2a70f7578f13
Gerrit-Change-Number: 18778
Gerrit-PatchSet: 1
Gerrit-Owner: [email protected]
Gerrit-MessageType: newchange
