This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit e298601bb7b8e2fda957074d854c3daa2e8efc78 Author: Ali Alsuliman <[email protected]> AuthorDate: Thu May 14 18:14:22 2020 -0700 [ASTERIXDB-2735][EXT] Support boolean value for properties of external data - user model changes: no - storage format changes: no - interface changes: no Details: The configuration properties of the external data in the DDL statements are supplied as key-value pairs of strings. The user should be able to supply normal boolean values for boolean properties such as "header" which tells whether the data source has a header or not (for delimited data such as CSV). Change-Id: I0470f7f0bbf7476e1fbd6b9f124355cf8b00c17c Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/6343 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Michael Blow <[email protected]> --- .../aws/s3/csv-header/query-dataset.001.ddl.sqlpp | 2 +- .../s3/csv-warnings/query-dataset.007.ddl.sqlpp | 2 +- .../s3/csv-warnings/query-dataset.010.ddl.sqlpp | 2 +- .../aws/s3/csv/csv/query-dataset.000.ddl.sqlpp | 2 +- .../aws/s3/csv/gz/query-dataset.000.ddl.sqlpp | 2 +- .../asterix/external/util/ExternalDataUtils.java | 22 ++++++++++++++-------- asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj | 4 +++- .../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj | 4 +++- 8 files changed, 25 insertions(+), 15 deletions(-) diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv-header/query-dataset.001.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv-header/query-dataset.001.ddl.sqlpp index 54022fc..cef7274 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv-header/query-dataset.001.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv-header/query-dataset.001.ddl.sqlpp @@ -33,5 +33,5 @@ CREATE EXTERNAL DATASET ds1(t1) USING S3 ( ("container"="playground"), ("definition"="data_dir"), ("format"="CSV"), -("header"="true") +("header"=true) ); \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv-warnings/query-dataset.007.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv-warnings/query-dataset.007.ddl.sqlpp index 047a540..6f96ecf 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv-warnings/query-dataset.007.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv-warnings/query-dataset.007.ddl.sqlpp @@ -29,5 +29,5 @@ CREATE EXTERNAL DATASET ds1(t1) USING S3 ( ("definition"="data_dir"), ("format"="CSV"), ("header"="false"), -("redact-warnings"="true") +("redact-warnings"="True") ); \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv-warnings/query-dataset.010.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv-warnings/query-dataset.010.ddl.sqlpp index 575d46f..47daa38 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv-warnings/query-dataset.010.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv-warnings/query-dataset.010.ddl.sqlpp @@ -29,6 +29,6 @@ CREATE EXTERNAL DATASET ds2(t2) USING S3 ( ("definition"="data_dir"), ("format"="CSV"), ("header"="true"), -("redact-warnings"="true"), +("redact-warnings"=True), ("null"="\\N") ); \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/csv/query-dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/csv/query-dataset.000.ddl.sqlpp index 54e0dc4..a63f3bc 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/csv/query-dataset.000.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/csv/query-dataset.000.ddl.sqlpp @@ -33,6 +33,6 @@ CREATE EXTERNAL DATASET test(test) USING S3 ( ("container"="playground"), ("definition"="csv-data/reviews/csv"), ("format"="Csv"), -("header"="false"), +("header"=False), ("null"="") ); \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/gz/query-dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/gz/query-dataset.000.ddl.sqlpp index 413987d..b8df075 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/gz/query-dataset.000.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/gz/query-dataset.000.ddl.sqlpp @@ -33,6 +33,6 @@ CREATE EXTERNAL DATASET test(test) USING S3 ( ("container"="playground"), ("definition"="csv-data/reviews/gz"), ("format"="Csv"), -("header"="false"), +("header"=false), ("null"="") ); \ No newline at end of file diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java index 8fb17bc..2ae2838 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java @@ -23,7 +23,6 @@ import static org.apache.asterix.external.util.ExternalDataConstants.KEY_ESCAPE; import static org.apache.asterix.external.util.ExternalDataConstants.KEY_QUOTE; import static org.apache.asterix.external.util.ExternalDataConstants.KEY_RECORD_END; import static org.apache.asterix.external.util.ExternalDataConstants.KEY_RECORD_START; -import static org.apache.asterix.external.util.ExternalDataConstants.KEY_REDACT_WARNINGS; import java.util.EnumMap; import java.util.Map; @@ -370,11 +369,10 @@ public class ExternalDataUtils { configuration.put(ExternalDataConstants.KEY_FORMAT, lowerCaseFormat); } } - // normalize the "header" parameter - paramValue = configuration.get(ExternalDataConstants.KEY_HEADER); - if (paramValue != null) { - configuration.put(ExternalDataConstants.KEY_HEADER, paramValue.toLowerCase().trim()); - } + // normalize "header" parameter + putToLowerIfExists(configuration, ExternalDataConstants.KEY_HEADER); + // normalize "redact-warnings" parameter + putToLowerIfExists(configuration, ExternalDataConstants.KEY_REDACT_WARNINGS); } /** @@ -395,9 +393,10 @@ public class ExternalDataUtils { char delimiter = validateGetDelimiter(configuration); validateGetQuote(configuration, delimiter); validateGetEscape(configuration); - String value = configuration.get(KEY_REDACT_WARNINGS); + String value = configuration.get(ExternalDataConstants.KEY_REDACT_WARNINGS); if (value != null && !isBoolean(value)) { - throw new RuntimeDataException(ErrorCode.INVALID_REQ_PARAM_VAL, KEY_REDACT_WARNINGS, value); + throw new RuntimeDataException(ErrorCode.INVALID_REQ_PARAM_VAL, ExternalDataConstants.KEY_REDACT_WARNINGS, + value); } } @@ -430,4 +429,11 @@ public class ExternalDataUtils { throw new RuntimeDataException(ErrorCode.INVALID_CHAR_LENGTH, parameterValue, parameterName); } } + + private static void putToLowerIfExists(Map<String, String> configuration, String key) { + String paramValue = configuration.get(key); + if (paramValue != null) { + configuration.put(key, paramValue.toLowerCase().trim()); + } + } } diff --git a/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj b/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj index 127dc62..4bf4ea4 100644 --- a/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj +++ b/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj @@ -1238,7 +1238,9 @@ Pair<String, String> KeyValuePair() throws ParseException: String value; } { - <LEFTPAREN> key = StringLiteral() <EQ> value = StringLiteral() <RIGHTPAREN> + <LEFTPAREN> key = StringLiteral() + <EQ>( value = StringLiteral() | (<TRUE> | <FALSE>) {value = token.image.toLowerCase();} ) + <RIGHTPAREN> { return new Pair<String, String>(key, value); } diff --git a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj index 107e240..7538887 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj +++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj @@ -1564,7 +1564,9 @@ Pair<String, String> KeyValuePair() throws ParseException: String value; } { - <LEFTPAREN> key = ConstantString() <EQ> value = ConstantString() <RIGHTPAREN> + <LEFTPAREN> key = ConstantString() + <EQ> ( value = ConstantString() | (<TRUE> | <FALSE>) {value = token.image.toLowerCase();} ) + <RIGHTPAREN> { return new Pair<String, String>(key, value); }
