This is an automated email from the ASF dual-hosted git repository. mhubail pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 325d345d1ca376358e785876fd3c6a50d161505d Author: Hussain Towaileb <[email protected]> AuthorDate: Thu Mar 27 01:03:27 2025 +0300 [NO ISSUE][EXT]: Centralize CSV properties in one place Ext-ref: MB-65990 Change-Id: I53f4584487f8706447ef97bf816c7e908bba7d2a Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19556 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Hussain Towaileb <[email protected]> Reviewed-by: Ali Alsuliman <[email protected]> --- .../csv/simple-csv/simple-csv.03.update.sqlpp | 2 +- .../csv/simple-csv/simple-csv.11.update.sqlpp | 2 +- .../apache/asterix/common/utils/CSVConstants.java} | 37 +++++++++------------- .../converter/CSVWithRecordConverterFactory.java | 7 ++-- .../reader/stream/QuotedLineRecordReader.java | 7 ++-- .../parser/factory/DelimitedDataParserFactory.java | 4 ++- .../external/util/ExternalDataConstants.java | 9 ------ .../asterix/external/util/ExternalDataUtils.java | 29 +++++++++-------- .../external/util/WriterValidationUtil.java | 12 ++++--- .../external/parser/test/RecordWithMetaTest.java | 7 ++-- .../test/StreamRecordReaderProviderTest.java | 4 ++- .../printers/csv/AObjectPrinterFactory.java | 17 +++++----- .../printers/csv/AStringPrinterFactory.java | 6 ++-- .../data/nontagged/printers/csv/CSVUtils.java | 21 ++++++------ .../nontagged/CSVPrinterFactoryProvider.java | 12 +++---- 15 files changed, 88 insertions(+), 88 deletions(-) diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/simple-csv/simple-csv.03.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/simple-csv/simple-csv.03.update.sqlpp index b9a6e9ad51..bd7403ed7d 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/simple-csv/simple-csv.03.update.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/simple-csv/simple-csv.03.update.sqlpp @@ -39,5 +39,5 @@ WITH { "quote":"'", "force-quote":"false", "escape":"\\", - "empty_field_as_null":"true" + "empty-string-as-null":"true" } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/simple-csv/simple-csv.11.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/simple-csv/simple-csv.11.update.sqlpp index 7e477bee79..b7028f5402 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/simple-csv/simple-csv.11.update.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/simple-csv/simple-csv.11.update.sqlpp @@ -40,5 +40,5 @@ WITH { "quote":"'", "force-quote":"false", "escape":"\\", - "empty_field_as_null":"true" + "empty-string-as-null":"true" } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/simple-csv/simple-csv.03.update.sqlpp b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/CSVConstants.java similarity index 55% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/simple-csv/simple-csv.03.update.sqlpp copy to asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/CSVConstants.java index b9a6e9ad51..74cc20f60a 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/simple-csv/simple-csv.03.update.sqlpp +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/CSVConstants.java @@ -16,28 +16,21 @@ * specific language governing permissions and limitations * under the License. */ +package org.apache.asterix.common.utils; -/* - * This test handles the case with all the CSV knobs. Schema type is JSON style. - */ +public class CSVConstants { + + private CSVConstants() { + } -USE test; + public static final String KEY_HEADER = "header"; + public static final String KEY_DELIMITER = "delimiter"; + public static final String KEY_RECORD_DELIMITER = "record-delimiter"; + public static final String KEY_QUOTE = "quote"; + public static final String KEY_FORCE_QUOTE = "force-quote"; + public static final String KEY_EMPTY_STRING_AS_NULL = "empty-string-as-null"; + public static final String KEY_ESCAPE = "escape"; -COPY ( - SELECT id, null name, amount, accountNumber FROM TestCollection -) toWriter -TO %adapter% -PATH (%pathprefix% "copy-to-result", "csv", "simple-csv", "1") -TYPE ( {id: bigint, name: string?, amount: float, accountNumber: double} ) -WITH { - %template_colons%, - %additionalProperties% - "format":"csv", - "delimiter":"|", - "header":"true", - "null":"IamNull", - "quote":"'", - "force-quote":"false", - "escape":"\\", - "empty_field_as_null":"true" -} \ No newline at end of file + // a string representing the NULL value + public static final String KEY_NULL_STR = "null"; +} diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/CSVWithRecordConverterFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/CSVWithRecordConverterFactory.java index 19615008c8..61b0f4ecd5 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/CSVWithRecordConverterFactory.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/CSVWithRecordConverterFactory.java @@ -18,6 +18,8 @@ */ package org.apache.asterix.external.input.record.converter; +import static org.apache.asterix.common.utils.CSVConstants.KEY_DELIMITER; + import java.util.Arrays; import java.util.Map; @@ -56,10 +58,9 @@ public class CSVWithRecordConverterFactory implements IRecordConverterFactory<ch "Unspecified " + ExternalDataConstants.KEY_RECORD_INDEX + " for csv to csv with record converter"); } recordIndex = Integer.parseInt(property); - property = configuration.get(ExternalDataConstants.KEY_DELIMITER); + property = configuration.get(KEY_DELIMITER); if (property == null) { - throw new AsterixException( - "Unspecified " + ExternalDataConstants.KEY_DELIMITER + " for csv to csv with record converter"); + throw new AsterixException("Unspecified " + KEY_DELIMITER + " for csv to csv with record converter"); } if (property.trim().length() > 1) { throw new AsterixException("Large delimiter. The maximum delimiter size = 1"); diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java index 2035a8e473..5fe0589dfb 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java @@ -18,6 +18,7 @@ */ package org.apache.asterix.external.input.record.reader.stream; +import static org.apache.asterix.common.utils.CSVConstants.KEY_QUOTE; import static org.apache.asterix.external.util.ExternalDataConstants.BYTE_ORDER_MARK; import static org.apache.asterix.external.util.ExternalDataConstants.REC_ENDED_AT_EOF; @@ -45,15 +46,15 @@ public class QuotedLineRecordReader extends LineRecordReader { private IWarningCollector warnings; private static final List<String> recordReaderFormats = Collections.unmodifiableList( Arrays.asList(ExternalDataConstants.FORMAT_DELIMITED_TEXT, ExternalDataConstants.FORMAT_CSV)); - private static final String REQUIRED_CONFIGS = ExternalDataConstants.KEY_QUOTE; + private static final String REQUIRED_CONFIGS = KEY_QUOTE; @Override public void configure(IHyracksTaskContext ctx, AsterixInputStream inputStream, Map<String, String> config) throws HyracksDataException { super.configure(ctx, inputStream, config); this.warnings = ctx.getWarningCollector(); - String quoteString = config.get(ExternalDataConstants.KEY_QUOTE); - ExternalDataUtils.validateChar(quoteString, ExternalDataConstants.KEY_QUOTE); + String quoteString = config.get(KEY_QUOTE); + ExternalDataUtils.validateChar(quoteString, KEY_QUOTE); this.quote = quoteString.charAt(0); this.escape = ExternalDataUtils.validateGetEscape(config, config.get(ExternalDataConstants.KEY_FORMAT)); } diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/factory/DelimitedDataParserFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/factory/DelimitedDataParserFactory.java index bad742e71a..65cf552fbc 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/factory/DelimitedDataParserFactory.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/factory/DelimitedDataParserFactory.java @@ -18,6 +18,8 @@ */ package org.apache.asterix.external.parser.factory; +import static org.apache.asterix.common.utils.CSVConstants.KEY_NULL_STR; + import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -53,7 +55,7 @@ public class DelimitedDataParserFactory extends AbstractRecordStreamParserFactor char escape = ExternalDataUtils.validateGetEscape(configuration, configuration.get(ExternalDataConstants.KEY_FORMAT)); boolean hasHeader = ExternalDataUtils.hasHeader(configuration); - String nullString = configuration.get(ExternalDataConstants.KEY_NULL_STR); + String nullString = configuration.get(KEY_NULL_STR); return new DelimitedDataParser(context, valueParserFactories, delimiter, quote, escape, hasHeader, recordType, ExternalDataUtils.getDataSourceType(configuration).equals(DataSourceType.STREAM), nullString); } diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java index f477b07812..d399b07c6a 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java @@ -73,10 +73,8 @@ public class ExternalDataConstants { //Base64 encoded function call information public static final String KEY_HADOOP_ASTERIX_FUNCTION_CALL_INFORMATION = "org.apache.asterix.function.info"; public static final String KEY_SOURCE_DATATYPE = "type-name"; - public static final String KEY_DELIMITER = "delimiter"; public static final String KEY_PARSER_FACTORY = "parser-factory"; public static final String KEY_DATA_PARSER = "parser"; - public static final String KEY_HEADER = "header"; public static final String KEY_READER = "reader"; public static final String KEY_READER_STREAM = "stream"; public static final String KEY_TYPE_NAME = "type-name"; @@ -92,11 +90,6 @@ public class ExternalDataConstants { public static final String PARQUET_DEFAULT_PAGE_SIZE = "8KB"; public static final String KEY_INCLUDE = "include"; public static final String KEY_EXCLUDE = "exclude"; - public static final String KEY_QUOTE = "quote"; - public static final String KEY_FORCE_QUOTE = "force-quote"; - public static final String KEY_EMPTY_FIELD_AS_NULL = "empty_field_as_null"; - public static final String KEY_RECORD_DELIMITER = "record-delimiter"; - public static final String KEY_ESCAPE = "escape"; public static final String KEY_PARSER = "parser"; public static final String KEY_DATASET_RECORD = "dataset-record"; public static final String KEY_RSS_URL = "url"; @@ -140,8 +133,6 @@ public class ExternalDataConstants { public static final String KEY_HTTP_PROXY_PORT = "http-proxy-port"; public static final String KEY_HTTP_PROXY_USER = "http-proxy-user"; public static final String KEY_HTTP_PROXY_PASSWORD = "http-proxy-password"; - // a string representing the NULL value - public static final String KEY_NULL_STR = "null"; public static final String KEY_REDACT_WARNINGS = "redact-warnings"; public static final String KEY_REQUESTED_FIELDS = "requested-fields"; public static final String KEY_EXTERNAL_SCAN_BUFFER_SIZE = "external-scan-buffer-size"; diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java index fac11990da..0fed43f7e3 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java @@ -19,14 +19,17 @@ package org.apache.asterix.external.util; import static org.apache.asterix.common.metadata.MetadataConstants.DEFAULT_DATABASE; +import static org.apache.asterix.common.utils.CSVConstants.KEY_DELIMITER; +import static org.apache.asterix.common.utils.CSVConstants.KEY_EMPTY_STRING_AS_NULL; +import static org.apache.asterix.common.utils.CSVConstants.KEY_ESCAPE; +import static org.apache.asterix.common.utils.CSVConstants.KEY_FORCE_QUOTE; +import static org.apache.asterix.common.utils.CSVConstants.KEY_HEADER; +import static org.apache.asterix.common.utils.CSVConstants.KEY_QUOTE; import static org.apache.asterix.external.util.ExternalDataConstants.DEFINITION_FIELD_NAME; -import static org.apache.asterix.external.util.ExternalDataConstants.KEY_DELIMITER; -import static org.apache.asterix.external.util.ExternalDataConstants.KEY_ESCAPE; import static org.apache.asterix.external.util.ExternalDataConstants.KEY_EXCLUDE; import static org.apache.asterix.external.util.ExternalDataConstants.KEY_EXTERNAL_SCAN_BUFFER_SIZE; import static org.apache.asterix.external.util.ExternalDataConstants.KEY_INCLUDE; import static org.apache.asterix.external.util.ExternalDataConstants.KEY_PATH; -import static org.apache.asterix.external.util.ExternalDataConstants.KEY_QUOTE; import static org.apache.asterix.external.util.ExternalDataConstants.KEY_RECORD_END; import static org.apache.asterix.external.util.ExternalDataConstants.KEY_RECORD_START; import static org.apache.asterix.external.util.aws.s3.S3AuthUtils.configureAwsS3HdfsJobConf; @@ -281,7 +284,7 @@ public class ExternalDataUtils { } public static boolean hasHeader(Map<String, String> configuration) { - return isTrue(configuration, ExternalDataConstants.KEY_HEADER); + return isTrue(configuration, KEY_HEADER); } public static boolean isTrue(Map<String, String> configuration, String key) { @@ -628,7 +631,7 @@ public class ExternalDataUtils { } } // normalize "header" parameter - putToLowerIfExists(configuration, ExternalDataConstants.KEY_HEADER); + putToLowerIfExists(configuration, KEY_HEADER); // normalize "redact-warnings" parameter putToLowerIfExists(configuration, ExternalDataConstants.KEY_REDACT_WARNINGS); } @@ -641,22 +644,20 @@ public class ExternalDataUtils { */ public static void validate(Map<String, String> configuration) throws HyracksDataException { String format = configuration.get(ExternalDataConstants.KEY_FORMAT); - String header = configuration.get(ExternalDataConstants.KEY_HEADER); - String forceQuote = configuration.get(ExternalDataConstants.KEY_FORCE_QUOTE); - String emptyFieldAsNull = configuration.get(ExternalDataConstants.KEY_EMPTY_FIELD_AS_NULL); + String header = configuration.get(KEY_HEADER); + String forceQuote = configuration.get(KEY_FORCE_QUOTE); + String emptyFieldAsNull = configuration.get(KEY_EMPTY_STRING_AS_NULL); if (format != null && isHeaderRequiredFor(format) && header == null) { - throw new RuntimeDataException(ErrorCode.PARAMETERS_REQUIRED, ExternalDataConstants.KEY_HEADER); + throw new RuntimeDataException(ErrorCode.PARAMETERS_REQUIRED, KEY_HEADER); } if (header != null && !isBoolean(header)) { - throw new RuntimeDataException(ErrorCode.INVALID_REQ_PARAM_VAL, ExternalDataConstants.KEY_HEADER, header); + throw new RuntimeDataException(ErrorCode.INVALID_REQ_PARAM_VAL, KEY_HEADER, header); } if (forceQuote != null && !isBoolean(forceQuote)) { - throw new RuntimeDataException(ErrorCode.INVALID_REQ_PARAM_VAL, ExternalDataConstants.KEY_FORCE_QUOTE, - forceQuote); + throw new RuntimeDataException(ErrorCode.INVALID_REQ_PARAM_VAL, KEY_FORCE_QUOTE, forceQuote); } if (emptyFieldAsNull != null && !isBoolean(emptyFieldAsNull)) { - throw new RuntimeDataException(ErrorCode.INVALID_REQ_PARAM_VAL, - ExternalDataConstants.KEY_EMPTY_FIELD_AS_NULL, emptyFieldAsNull); + throw new RuntimeDataException(ErrorCode.INVALID_REQ_PARAM_VAL, KEY_EMPTY_STRING_AS_NULL, emptyFieldAsNull); } char delimiter = validateGetDelimiter(configuration); validateGetQuote(configuration, delimiter); diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/WriterValidationUtil.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/WriterValidationUtil.java index f43a2a7adc..92afd40034 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/WriterValidationUtil.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/WriterValidationUtil.java @@ -22,6 +22,10 @@ import static org.apache.asterix.common.exceptions.ErrorCode.INVALID_REQ_PARAM_V import static org.apache.asterix.common.exceptions.ErrorCode.MAXIMUM_VALUE_ALLOWED_FOR_PARAM; import static org.apache.asterix.common.exceptions.ErrorCode.MINIMUM_VALUE_ALLOWED_FOR_PARAM; import static org.apache.asterix.common.exceptions.ErrorCode.PARAMETERS_REQUIRED; +import static org.apache.asterix.common.utils.CSVConstants.KEY_DELIMITER; +import static org.apache.asterix.common.utils.CSVConstants.KEY_ESCAPE; +import static org.apache.asterix.common.utils.CSVConstants.KEY_QUOTE; +import static org.apache.asterix.common.utils.CSVConstants.KEY_RECORD_DELIMITER; import static org.apache.asterix.external.util.ExternalDataConstants.FORMAT_CSV; import static org.apache.asterix.external.util.ExternalDataConstants.FORMAT_JSON_LOWER_CASE; import static org.apache.asterix.external.util.ExternalDataConstants.FORMAT_PARQUET; @@ -57,7 +61,7 @@ public class WriterValidationUtil { private static void validateQuote(Map<String, String> configuration, SourceLocation sourceLocation) throws CompilationException { - String quote = configuration.get(ExternalDataConstants.KEY_QUOTE); + String quote = configuration.get(KEY_QUOTE); if (quote != null && !ExternalDataConstants.WRITER_SUPPORTED_QUOTES.contains(quote.toLowerCase())) { throw CompilationException.create(ErrorCode.CSV_INVALID_QUOTE, sourceLocation, quote, ExternalDataConstants.WRITER_SUPPORTED_QUOTES.toString()); @@ -251,19 +255,19 @@ public class WriterValidationUtil { private static void validateDelimiter(Map<String, String> configuration, SourceLocation sourceLocation) throws CompilationException { - String delimiter = configuration.get(ExternalDataConstants.KEY_DELIMITER); + String delimiter = configuration.get(KEY_DELIMITER); unitByteCondition(delimiter, sourceLocation, ErrorCode.INVALID_DELIMITER); } private static void validateEscape(Map<String, String> configuration, SourceLocation sourceLocation) throws CompilationException { - String escape = configuration.get(ExternalDataConstants.KEY_ESCAPE); + String escape = configuration.get(KEY_ESCAPE); unitByteCondition(escape, sourceLocation, ErrorCode.CSV_INVALID_ESCAPE); } private static void validateRecordDelimiter(Map<String, String> configuration, SourceLocation sourceLocation) throws CompilationException { - String recordDel = configuration.get(ExternalDataConstants.KEY_RECORD_DELIMITER); + String recordDel = configuration.get(KEY_RECORD_DELIMITER); unitByteCondition(recordDel, sourceLocation, ErrorCode.CSV_INVALID_FORCE_QUOTE); } diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java index 3eb1bd4a3e..4f69873980 100644 --- a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java +++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java @@ -18,6 +18,9 @@ */ package org.apache.asterix.external.parser.test; +import static org.apache.asterix.common.utils.CSVConstants.KEY_HEADER; +import static org.apache.asterix.common.utils.CSVConstants.KEY_QUOTE; + import java.io.File; import java.io.PrintStream; import java.nio.file.Files; @@ -91,8 +94,8 @@ public class RecordWithMetaTest { LocalFSInputStream inputStream = new LocalFSInputStream(watcher); // create reader record reader Map<String, String> config = new HashMap<>(); - config.put(ExternalDataConstants.KEY_HEADER, "true"); - config.put(ExternalDataConstants.KEY_QUOTE, ExternalDataConstants.DEFAULT_QUOTE); + config.put(KEY_HEADER, "true"); + config.put(KEY_QUOTE, ExternalDataConstants.DEFAULT_QUOTE); LineRecordReader lineReader = new LineRecordReader(); lineReader.configure(ctx, inputStream, config); // create csv with json record reader diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/StreamRecordReaderProviderTest.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/StreamRecordReaderProviderTest.java index 7267357ac6..b51cdbe12b 100644 --- a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/StreamRecordReaderProviderTest.java +++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/StreamRecordReaderProviderTest.java @@ -18,6 +18,8 @@ */ package org.apache.asterix.external.parser.test; +import static org.apache.asterix.common.utils.CSVConstants.KEY_QUOTE; + import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -46,7 +48,7 @@ public class StreamRecordReaderProviderTest { } config.clear(); config.put(ExternalDataConstants.KEY_FORMAT, ExternalDataConstants.FORMAT_CSV); - config.put(ExternalDataConstants.KEY_QUOTE, "\u0000"); + config.put(KEY_QUOTE, "\u0000"); Assert.assertTrue(StreamRecordReaderProvider.getRecordReaderClazz(config) != null); } } diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/AObjectPrinterFactory.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/AObjectPrinterFactory.java index 05babc537b..2f4d18024f 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/AObjectPrinterFactory.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/AObjectPrinterFactory.java @@ -18,13 +18,13 @@ */ package org.apache.asterix.dataflow.data.nontagged.printers.csv; +import static org.apache.asterix.common.utils.CSVConstants.KEY_DELIMITER; +import static org.apache.asterix.common.utils.CSVConstants.KEY_EMPTY_STRING_AS_NULL; +import static org.apache.asterix.common.utils.CSVConstants.KEY_ESCAPE; +import static org.apache.asterix.common.utils.CSVConstants.KEY_FORCE_QUOTE; +import static org.apache.asterix.common.utils.CSVConstants.KEY_NULL_STR; +import static org.apache.asterix.common.utils.CSVConstants.KEY_QUOTE; import static org.apache.asterix.dataflow.data.nontagged.printers.csv.CSVUtils.DEFAULT_VALUES; -import static org.apache.asterix.dataflow.data.nontagged.printers.csv.CSVUtils.KEY_DELIMITER; -import static org.apache.asterix.dataflow.data.nontagged.printers.csv.CSVUtils.KEY_EMPTY_FIELD_AS_NULL; -import static org.apache.asterix.dataflow.data.nontagged.printers.csv.CSVUtils.KEY_ESCAPE; -import static org.apache.asterix.dataflow.data.nontagged.printers.csv.CSVUtils.KEY_FORCE_QUOTE; -import static org.apache.asterix.dataflow.data.nontagged.printers.csv.CSVUtils.KEY_NULL; -import static org.apache.asterix.dataflow.data.nontagged.printers.csv.CSVUtils.KEY_QUOTE; import static org.apache.asterix.dataflow.data.nontagged.printers.csv.CSVUtils.getCharOrDefault; import java.io.PrintStream; @@ -58,8 +58,9 @@ public class AObjectPrinterFactory implements IPrinterFactory { private AObjectPrinterFactory(ARecordType itemType, Map<String, String> configuration) { this.itemType = itemType; this.configuration = configuration; - this.emptyFieldAsNull = Boolean.parseBoolean(configuration.get(KEY_EMPTY_FIELD_AS_NULL)); - this.nullString = configuration.get(KEY_NULL) != null ? configuration.get(KEY_NULL) : DEFAULT_NULL_STRING; + this.emptyFieldAsNull = Boolean.parseBoolean(configuration.get(KEY_EMPTY_STRING_AS_NULL)); + this.nullString = + configuration.get(KEY_NULL_STR) != null ? configuration.get(KEY_NULL_STR) : DEFAULT_NULL_STRING; this.forceQuote = Boolean.parseBoolean(configuration.get(KEY_FORCE_QUOTE)); this.quote = getCharOrDefault(configuration.get(KEY_QUOTE), DEFAULT_VALUES.get(KEY_QUOTE)); this.escape = getCharOrDefault(configuration.get(KEY_ESCAPE), DEFAULT_VALUES.get(KEY_ESCAPE)); diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/AStringPrinterFactory.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/AStringPrinterFactory.java index 99b4c08d3a..e18d450160 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/AStringPrinterFactory.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/AStringPrinterFactory.java @@ -18,10 +18,10 @@ */ package org.apache.asterix.dataflow.data.nontagged.printers.csv; +import static org.apache.asterix.common.utils.CSVConstants.KEY_DELIMITER; +import static org.apache.asterix.common.utils.CSVConstants.KEY_ESCAPE; +import static org.apache.asterix.common.utils.CSVConstants.KEY_QUOTE; import static org.apache.asterix.dataflow.data.nontagged.printers.csv.CSVUtils.DEFAULT_VALUES; -import static org.apache.asterix.dataflow.data.nontagged.printers.csv.CSVUtils.KEY_DELIMITER; -import static org.apache.asterix.dataflow.data.nontagged.printers.csv.CSVUtils.KEY_ESCAPE; -import static org.apache.asterix.dataflow.data.nontagged.printers.csv.CSVUtils.KEY_QUOTE; import static org.apache.asterix.dataflow.data.nontagged.printers.csv.CSVUtils.getCharOrDefault; import java.io.PrintStream; diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/CSVUtils.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/CSVUtils.java index e06309e25f..37d4dbd21a 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/CSVUtils.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/CSVUtils.java @@ -18,6 +18,15 @@ */ package org.apache.asterix.dataflow.data.nontagged.printers.csv; +import static org.apache.asterix.common.utils.CSVConstants.KEY_DELIMITER; +import static org.apache.asterix.common.utils.CSVConstants.KEY_EMPTY_STRING_AS_NULL; +import static org.apache.asterix.common.utils.CSVConstants.KEY_ESCAPE; +import static org.apache.asterix.common.utils.CSVConstants.KEY_FORCE_QUOTE; +import static org.apache.asterix.common.utils.CSVConstants.KEY_HEADER; +import static org.apache.asterix.common.utils.CSVConstants.KEY_NULL_STR; +import static org.apache.asterix.common.utils.CSVConstants.KEY_QUOTE; +import static org.apache.asterix.common.utils.CSVConstants.KEY_RECORD_DELIMITER; + import java.io.IOException; import java.io.PrintStream; import java.util.EnumSet; @@ -37,14 +46,6 @@ public class CSVUtils { // Constants for the supported CSV parameters public static final String NONE = "none"; - public static final String KEY_NULL = "null"; - public static final String KEY_ESCAPE = "escape"; - public static final String KEY_HEADER = "header"; - public static final String KEY_DELIMITER = "delimiter"; - public static final String KEY_RECORD_DELIMITER = "recordDelimiter"; - public static final String KEY_FORCE_QUOTE = "forceQuote"; - public static final String KEY_QUOTE = "quote"; - public static final String KEY_EMPTY_FIELD_AS_NULL = "empty_field_as_null"; public static final char DEFAULT_QUOTE = '"'; private static final String DEFAULT_DELIMITER_VALUE = ","; private static final String DEFAULT_NULL_VALUE = ""; @@ -58,14 +59,14 @@ public class CSVUtils { static { DEFAULT_VALUES = new HashMap<>(); - DEFAULT_VALUES.put(KEY_NULL, DEFAULT_NULL_VALUE); + DEFAULT_VALUES.put(KEY_NULL_STR, DEFAULT_NULL_VALUE); DEFAULT_VALUES.put(KEY_ESCAPE, DOUBLE_QUOTES); DEFAULT_VALUES.put(KEY_HEADER, FALSE); DEFAULT_VALUES.put(KEY_DELIMITER, DEFAULT_DELIMITER_VALUE); DEFAULT_VALUES.put(KEY_RECORD_DELIMITER, DEFAULT_RECORD_DELIMITER); DEFAULT_VALUES.put(KEY_FORCE_QUOTE, FALSE); DEFAULT_VALUES.put(KEY_QUOTE, DOUBLE_QUOTES); - DEFAULT_VALUES.put(KEY_EMPTY_FIELD_AS_NULL, FALSE); + DEFAULT_VALUES.put(KEY_EMPTY_STRING_AS_NULL, FALSE); } public static boolean isEmptyString(byte[] b, int s, int l) { diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/CSVPrinterFactoryProvider.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/CSVPrinterFactoryProvider.java index 054f9c7f1a..4d28ce4d1c 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/CSVPrinterFactoryProvider.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/CSVPrinterFactoryProvider.java @@ -18,11 +18,11 @@ */ package org.apache.asterix.formats.nontagged; -import static org.apache.asterix.dataflow.data.nontagged.printers.csv.CSVUtils.KEY_DELIMITER; -import static org.apache.asterix.dataflow.data.nontagged.printers.csv.CSVUtils.KEY_ESCAPE; -import static org.apache.asterix.dataflow.data.nontagged.printers.csv.CSVUtils.KEY_FORCE_QUOTE; -import static org.apache.asterix.dataflow.data.nontagged.printers.csv.CSVUtils.KEY_NULL; -import static org.apache.asterix.dataflow.data.nontagged.printers.csv.CSVUtils.KEY_QUOTE; +import static org.apache.asterix.common.utils.CSVConstants.KEY_DELIMITER; +import static org.apache.asterix.common.utils.CSVConstants.KEY_ESCAPE; +import static org.apache.asterix.common.utils.CSVConstants.KEY_FORCE_QUOTE; +import static org.apache.asterix.common.utils.CSVConstants.KEY_NULL_STR; +import static org.apache.asterix.common.utils.CSVConstants.KEY_QUOTE; import java.util.Collections; import java.util.Map; @@ -99,7 +99,7 @@ public class CSVPrinterFactoryProvider implements IPrinterFactoryProvider { return AInt64PrinterFactory.INSTANCE; case MISSING: case NULL: - return ANullPrinterFactory.createInstance(configuration.get(KEY_NULL)); + return ANullPrinterFactory.createInstance(configuration.get(KEY_NULL_STR)); case BOOLEAN: return ABooleanPrinterFactory.INSTANCE; case FLOAT:
