This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 1bea9e9c77b451c7732b43a3a79fd15bad1c26c0 Author: Hussain Towaileb <[email protected]> AuthorDate: Thu Mar 27 05:18:46 2025 +0300 [ASTERIXDB-3604][EXT]: Add support to datetime format Details: - Add support to writing date/time formats for COPY TO CSV. - Add test cases. Ext-ref: MB-65784 Change-Id: I6f29536e962399cbffcd377fa346eea7ddfc082a Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19697 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Hussain Towaileb <[email protected]> Reviewed-by: Peeyush Gupta <[email protected]> --- .../asterix/translator/CompiledStatements.java | 10 +- .../translator/LangExpressionToPlanTranslator.java | 2 +- .../copy-to/csv/datetime/test.000.ddl.sqlpp} | 12 +-- .../copy-to/csv/datetime/test.030.update.sqlpp} | 29 +++--- .../copy-to/csv/datetime/test.031.update.sqlpp} | 30 +++--- .../copy-to/csv/datetime/test.032.update.sqlpp} | 30 +++--- .../copy-to/csv/datetime/test.033.update.sqlpp} | 31 +++--- .../copy-to/csv/datetime/test.034.update.sqlpp} | 30 +++--- .../copy-to/csv/datetime/test.035.update.sqlpp} | 30 +++--- .../copy-to/csv/datetime/test.036.update.sqlpp} | 32 +++--- .../copy-to/csv/datetime/test.040.ddl.sqlpp | 107 ++++++++++++++++++++ .../copy-to/csv/datetime/test.130.query.sqlpp} | 18 ++-- .../copy-to/csv/datetime/test.131.query.sqlpp} | 16 +-- .../copy-to/csv/datetime/test.132.query.sqlpp} | 17 +--- .../copy-to/csv/datetime/test.133.query.sqlpp} | 16 +-- .../copy-to/csv/datetime/test.134.query.sqlpp} | 16 +-- .../copy-to/csv/datetime/test.135.query.sqlpp} | 16 +-- .../copy-to/csv/datetime/test.136.query.sqlpp} | 18 ++-- .../copy-to/csv/datetime/test.999.ddl.sqlpp} | 12 +-- .../results/copy-to/csv/datetime/result.130.adm | 1 + .../results/copy-to/csv/datetime/result.131.adm | 1 + .../results/copy-to/csv/datetime/result.132.adm | 1 + .../results/copy-to/csv/datetime/result.133.adm | 1 + .../results/copy-to/csv/datetime/result.134.adm | 1 + .../results/copy-to/csv/datetime/result.135.adm | 1 + .../results/copy-to/csv/datetime/result.136.adm | 1 + .../runtimets/testsuite_external_dataset_s3.xml | 10 ++ .../lang/common/statement/CopyToStatement.java | 20 ++-- .../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj | 15 ++- .../metadata/declared/IExternalWriteDataSink.java | 4 + .../asterix/metadata/declared/WriteDataSink.java | 9 +- .../metadata/provider/ExternalWriterProvider.java | 3 +- .../data/nontagged/printers/PrintTools.java | 20 +++- .../printers/csv/ADatePrinterFactory.java | 8 +- .../printers/csv/ADateTimePrinterFactory.java | 8 +- .../printers/csv/AObjectPrinterFactory.java | 109 +++++++++++++++++++-- .../printers/csv/ARecordPrinterFactory.java | 7 +- .../printers/csv/ATimePrinterFactory.java | 8 +- .../nontagged/CSVPrinterFactoryProvider.java | 14 +-- .../om/pointables/printer/csv/APrintVisitor.java | 7 +- 40 files changed, 461 insertions(+), 260 deletions(-) diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java index 6813e84e3b..7c9b548cb6 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java @@ -34,7 +34,6 @@ import org.apache.asterix.metadata.entities.Dataset; import org.apache.asterix.metadata.entities.Datatype; import org.apache.asterix.metadata.entities.Index; import org.apache.asterix.om.types.ARecordType; -import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.api.exceptions.SourceLocation; /** @@ -528,7 +527,7 @@ public class CompiledStatements { return condition; } - public Query getQuery() throws AlgebricksException { + public Query getQuery() { return query; } @@ -605,9 +604,9 @@ public class CompiledStatements { private final List<OrderbyClause.NullOrderModifier> orderByNullModifierList; private final List<Expression> keyExpressions; private final boolean autogenerated; - private final ARecordType itemType; private final ARecordType parquetSchema; + private final Map<String, String> formatConfigs; public CompiledCopyToStatement(CopyToStatement copyToStatement) { this.query = copyToStatement.getQuery(); @@ -625,6 +624,7 @@ public class CompiledStatements { this.autogenerated = copyToStatement.isAutogenerated(); this.itemType = eddDecl.getItemType(); this.parquetSchema = eddDecl.getParquetSchema(); + this.formatConfigs = copyToStatement.getFormatConfigs(); } @Override @@ -696,6 +696,10 @@ public class CompiledStatements { return autogenerated; } + public Map<String, String> getFormatConfigs() { + return formatConfigs; + } + public boolean isFileStoreSink() { return keyExpressions.isEmpty() && !autogenerated; } diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java index 3bb871b0c8..e3e78e0c4f 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java @@ -467,7 +467,7 @@ abstract class LangExpressionToPlanTranslator // Write adapter configuration WriteDataSink writeDataSink = new WriteDataSink(copyTo.getAdapter(), copyTo.getProperties(), - copyTo.getItemType(), copyTo.getParquetSchema(), expr.getSourceLocation()); + copyTo.getItemType(), copyTo.getParquetSchema(), copyTo.getFormatConfigs(), expr.getSourceLocation()); // writeOperator WriteOperator writeOperator = new WriteOperator(sourceExprRef, new MutableObject<>(fullPathExpr), diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.000.ddl.sqlpp similarity index 67% copy from asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.000.ddl.sqlpp index 1168ba1de7..c04056a9e1 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.000.ddl.sqlpp @@ -17,16 +17,8 @@ * under the License. */ -package org.apache.asterix.metadata.declared; +DROP DATAVERSE test IF EXISTS; +CREATE DATAVERSE test; -import org.apache.asterix.om.types.ARecordType; -import org.apache.hyracks.algebricks.core.algebra.metadata.IWriteDataSink; -import org.apache.hyracks.api.exceptions.SourceLocation; -public interface IExternalWriteDataSink extends IWriteDataSink { - ARecordType getItemType(); - ARecordType getParquetSchema(); - - SourceLocation getSourceLoc(); -} diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.030.update.sqlpp similarity index 54% copy from asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.030.update.sqlpp index 1168ba1de7..4372e59425 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.030.update.sqlpp @@ -4,7 +4,7 @@ * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance + * "License", you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 @@ -17,16 +17,17 @@ * under the License. */ -package org.apache.asterix.metadata.declared; - -import org.apache.asterix.om.types.ARecordType; -import org.apache.hyracks.algebricks.core.algebra.metadata.IWriteDataSink; -import org.apache.hyracks.api.exceptions.SourceLocation; - -public interface IExternalWriteDataSink extends IWriteDataSink { - ARecordType getItemType(); - - ARecordType getParquetSchema(); - - SourceLocation getSourceLoc(); -} +COPY ( + select value x FROM [ + {"id":1, "name":"", "amount":123.2, "accountNumber":345.34, "joinDatetime": datetime("2025-04-25T14:53:54.398"), "joinDate": date("2025-04-25"), "joinTime": time("14:53:54.398")} + ] as x +) toWrite +TO %adapter% +PATH (%pathprefix% "copy-to-result", "csv", "simple-csv", "datetime-not-formatted") +TYPE ( {id: bigint, name: string?, amount: float, accountNumber: double, joinDatetime: datetime, joinDate: date, joinTime: time} ) +WITH { + %template_colons%, + %additionalProperties% + "format":"csv", + "header":"true" +} \ No newline at end of file diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.031.update.sqlpp similarity index 57% copy from asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.031.update.sqlpp index 1168ba1de7..8adfd8cb42 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.031.update.sqlpp @@ -4,7 +4,7 @@ * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance + * "License", you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 @@ -17,16 +17,18 @@ * under the License. */ -package org.apache.asterix.metadata.declared; - -import org.apache.asterix.om.types.ARecordType; -import org.apache.hyracks.algebricks.core.algebra.metadata.IWriteDataSink; -import org.apache.hyracks.api.exceptions.SourceLocation; - -public interface IExternalWriteDataSink extends IWriteDataSink { - ARecordType getItemType(); - - ARecordType getParquetSchema(); - - SourceLocation getSourceLoc(); -} +COPY ( + select value x FROM [ + {"id":1, "name":"", "amount":123.2, "accountNumber":345.34, "joinDatetime": datetime("2025-04-25T14:53:54.398")} + ] as x +) toWrite +TO %adapter% +PATH (%pathprefix% "copy-to-result", "csv", "simple-csv", "datetime-formatted") +TYPE ( {id: bigint, name: string?, amount: float, accountNumber: double, joinDatetime: datetime} ) +DATETIME "DD-MM-YYYYTss:mm:hh" +WITH { + %template_colons%, + %additionalProperties% + "format":"csv", + "header":"true" +} \ No newline at end of file diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.032.update.sqlpp similarity index 54% copy from asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.032.update.sqlpp index 1168ba1de7..38b0eefc24 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.032.update.sqlpp @@ -4,7 +4,7 @@ * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance + * "License", you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 @@ -17,16 +17,18 @@ * under the License. */ -package org.apache.asterix.metadata.declared; - -import org.apache.asterix.om.types.ARecordType; -import org.apache.hyracks.algebricks.core.algebra.metadata.IWriteDataSink; -import org.apache.hyracks.api.exceptions.SourceLocation; - -public interface IExternalWriteDataSink extends IWriteDataSink { - ARecordType getItemType(); - - ARecordType getParquetSchema(); - - SourceLocation getSourceLoc(); -} +COPY ( + select value x FROM [ + {"id":1, "name":"", "amount":123.2, "accountNumber":345.34, "joinDatetime": datetime("2025-04-25T14:53:54.398"), "joinDate": date("2025-04-25")} + ] as x +) toWrite +TO %adapter% +PATH (%pathprefix% "copy-to-result", "csv", "simple-csv", "datetime-date-formatted") +TYPE ( {id: bigint, name: string?, amount: float, accountNumber: double, joinDatetime: datetime, joinDate: date} ) +DATE "DD-MM-YYYY" DATETIME "DD-MM-YYYY" +WITH { + %template_colons%, + %additionalProperties% + "format":"csv", + "header":"true" +} \ No newline at end of file diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.033.update.sqlpp similarity index 54% copy from asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.033.update.sqlpp index 1168ba1de7..5628b8ec39 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.033.update.sqlpp @@ -4,7 +4,7 @@ * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance + * "License", you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 @@ -17,16 +17,19 @@ * under the License. */ -package org.apache.asterix.metadata.declared; - -import org.apache.asterix.om.types.ARecordType; -import org.apache.hyracks.algebricks.core.algebra.metadata.IWriteDataSink; -import org.apache.hyracks.api.exceptions.SourceLocation; - -public interface IExternalWriteDataSink extends IWriteDataSink { - ARecordType getItemType(); - - ARecordType getParquetSchema(); - - SourceLocation getSourceLoc(); -} +COPY ( + select value x FROM [ + {"id":1, "name":"", "amount":123.2, "accountNumber":345.34, "joinDatetime": datetime("2025-04-25T14:53:54.398"), "joinTime": time("14:53:54.398")} + ] as x +) toWrite +TO %adapter% +PATH (%pathprefix% "copy-to-result", "csv", "simple-csv", "datetime-time-formatted") +TYPE ( {id: bigint, name: string?, amount: float, accountNumber: double, joinDatetime: datetime, joinTime: time} ) +DATETIME "hh:mm:ss.nnna" +TIME "hh:mm:ss.nnna" +WITH { + %template_colons%, + %additionalProperties% + "format":"csv", + "header":"true" +} \ No newline at end of file diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.034.update.sqlpp similarity index 57% copy from asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.034.update.sqlpp index 1168ba1de7..d5044cd836 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.034.update.sqlpp @@ -4,7 +4,7 @@ * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance + * "License", you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 @@ -17,16 +17,18 @@ * under the License. */ -package org.apache.asterix.metadata.declared; - -import org.apache.asterix.om.types.ARecordType; -import org.apache.hyracks.algebricks.core.algebra.metadata.IWriteDataSink; -import org.apache.hyracks.api.exceptions.SourceLocation; - -public interface IExternalWriteDataSink extends IWriteDataSink { - ARecordType getItemType(); - - ARecordType getParquetSchema(); - - SourceLocation getSourceLoc(); -} +COPY ( + select value x FROM [ + {"id":1, "name":"", "amount":123.2, "accountNumber":345.34, "joinDatetime": datetime("2025-04-25T14:53:54.398")} + ] as x +) toWrite +TO %adapter% +PATH (%pathprefix% "copy-to-result", "csv", "simple-csv", "datetime-formatted-2") +TYPE ( {id: bigint, name: string?, amount: float, accountNumber: double, joinDatetime: datetime} ) +DATETIME "DD/MM/YYYY ss:mm:hh a" +WITH { + %template_colons%, + %additionalProperties% + "format":"csv", + "header":"true" +} \ No newline at end of file diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.035.update.sqlpp similarity index 56% copy from asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.035.update.sqlpp index 1168ba1de7..a80fd434f9 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.035.update.sqlpp @@ -4,7 +4,7 @@ * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance + * "License", you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 @@ -17,16 +17,18 @@ * under the License. */ -package org.apache.asterix.metadata.declared; - -import org.apache.asterix.om.types.ARecordType; -import org.apache.hyracks.algebricks.core.algebra.metadata.IWriteDataSink; -import org.apache.hyracks.api.exceptions.SourceLocation; - -public interface IExternalWriteDataSink extends IWriteDataSink { - ARecordType getItemType(); - - ARecordType getParquetSchema(); - - SourceLocation getSourceLoc(); -} +COPY ( + select value x FROM [ + {"id":1, "name":"", "amount":123.2, "accountNumber":345.34, "joinDatetime": datetime("2025-04-25T14:53:54.398")} + ] as x +) toWrite +TO %adapter% +PATH (%pathprefix% "copy-to-result", "csv", "simple-csv", "datetime-formatted-with-comma") +TYPE ( {id: bigint, name: string?, amount: float, accountNumber: double, joinDatetime: datetime} ) +DATETIME "DD/MMM/YYYY, ss:mm:hh a" +WITH { + %template_colons%, + %additionalProperties% + "format":"csv", + "header":"true" +} \ No newline at end of file diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.036.update.sqlpp similarity index 51% copy from asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.036.update.sqlpp index 1168ba1de7..26fb57cad2 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.036.update.sqlpp @@ -4,7 +4,7 @@ * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance + * "License", you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 @@ -17,16 +17,20 @@ * under the License. */ -package org.apache.asterix.metadata.declared; - -import org.apache.asterix.om.types.ARecordType; -import org.apache.hyracks.algebricks.core.algebra.metadata.IWriteDataSink; -import org.apache.hyracks.api.exceptions.SourceLocation; - -public interface IExternalWriteDataSink extends IWriteDataSink { - ARecordType getItemType(); - - ARecordType getParquetSchema(); - - SourceLocation getSourceLoc(); -} +COPY ( + select value x FROM [ + {"id":1, "name":"", "amount":123.2, "accountNumber":345.34, "joinDatetime": datetime("2025-04-25T14:53:54.398"), "joinDate": date("2025-04-25"), "joinTime": time("14:53:54.398")} + ] as x +) toWrite +TO %adapter% +PATH (%pathprefix% "copy-to-result", "csv", "simple-csv", "datetime-date-time-formatted-with-comma") +TYPE ( {id: bigint, name: string?, amount: float, accountNumber: double, joinDatetime: datetime, joinDate: date, joinTime: time} ) +DATETIME "DD/MMM/YYYY, ss:mm:hh a" +TIME "ss:mm:hh a" +DATE "DD/MMM/YYYY" +WITH { + %template_colons%, + %additionalProperties% + "format":"csv", + "header":"true" +} \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.040.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.040.ddl.sqlpp new file mode 100644 index 0000000000..e8dc8bd5a3 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.040.ddl.sqlpp @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; + +drop dataset datetimeNotFormattedDataset if exists; +drop dataset datetimeFormattedDataset if exists; +drop dataset datetimeDateFormattedDataset if exists; +drop dataset datatimeTimeFormattedDataset if exists; +drop dataset datetimeFormattedDataset2 if exists; +drop dataset datetimeFormattedWithCommaDataset if exists; +drop dataset datetimeDateTimeFormattedWithCommaDataset if exists; + +drop type datetimeNotFormattedType if exists; +drop type datetimeFormattedType if exists; +drop type datetimeDateFormattedType if exists; +drop type datatimeTimeFormattedType if exists; +drop type datetimeFormattedType2 if exists; +drop type datetimeFormattedWithCommaType if exists; +drop type datetimeDateTimeFormattedWithCommaType if exists; + +CREATE TYPE datetimeNotFormattedType AS { id: bigint, name: string?, amount: float, accountNumber: double, joinDateTime: string, joinDate: string, joinTime: string }; +CREATE TYPE datetimeFormattedType AS { id: bigint, name: string?, amount: float, accountNumber: double, joinDateTime: string }; +CREATE TYPE datetimeDateFormattedType AS { id: bigint, name: string?, amount: float, accountNumber: double, joinDateTime: string, joinDate: string }; +CREATE TYPE datatimeTimeFormattedType AS { id: bigint, name: string?, amount: float, accountNumber: double, joinDateTime: string, joinTime: string }; +CREATE TYPE datetimeFormattedType2 AS { id: bigint, name: string?, amount: float, accountNumber: double, joinDateTime: string }; +CREATE TYPE datetimeFormattedWithCommaType AS { id: bigint, name: string?, amount: float, accountNumber: double, joinDateTime: string }; +CREATE TYPE datetimeDateTimeFormattedWithCommaType AS { id: bigint, name: string?, amount: float, accountNumber: double, joinDateTime: string, joinDate: string, joinTime: string }; + +CREATE EXTERNAL DATASET datetimeNotFormattedDataset(datetimeNotFormattedType) USING %adapter% +( + %template%, + %additional_Properties%, + ("header"="true"), + ("definition"="%path_prefix%copy-to-result/csv/simple-csv/datetime-not-formatted"), + ("format" = "csv") +); + +CREATE EXTERNAL DATASET datetimeFormattedDataset(datetimeFormattedType) USING %adapter% +( + %template%, + %additional_Properties%, + ("header"="true"), + ("definition"="%path_prefix%copy-to-result/csv/simple-csv/datetime-formatted"), + ("format" = "csv") +); + +CREATE EXTERNAL DATASET datetimeDateFormattedDataset(datetimeDateFormattedType) USING %adapter% +( + %template%, + %additional_Properties%, + ("header"="true"), + ("definition"="%path_prefix%copy-to-result/csv/simple-csv/datetime-date-formatted"), + ("format" = "csv") +); + +CREATE EXTERNAL DATASET datatimeTimeFormattedDataset(datatimeTimeFormattedType) USING %adapter% +( + %template%, + %additional_Properties%, + ("header"="true"), + ("definition"="%path_prefix%copy-to-result/csv/simple-csv/datetime-time-formatted"), + ("format" = "csv") +); + +CREATE EXTERNAL DATASET datetimeFormattedDataset2(datetimeFormattedType2) USING %adapter% +( + %template%, + %additional_Properties%, + ("header"="true"), + ("definition"="%path_prefix%copy-to-result/csv/simple-csv/datetime-formatted-2"), + ("format" = "csv") +); + +CREATE EXTERNAL DATASET datetimeFormattedWithCommaDataset(datetimeFormattedWithCommaType) USING %adapter% +( + %template%, + %additional_Properties%, + ("header"="true"), + ("definition"="%path_prefix%copy-to-result/csv/simple-csv/datetime-formatted-with-comma"), + ("format" = "csv") +); + +CREATE EXTERNAL DATASET datetimeDateTimeFormattedWithCommaDataset(datetimeDateTimeFormattedWithCommaType) USING %adapter% +( + %template%, + %additional_Properties%, + ("header"="true"), + ("definition"="%path_prefix%copy-to-result/csv/simple-csv/datetime-date-time-formatted-with-comma"), + ("format" = "csv") +); \ No newline at end of file diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.130.query.sqlpp similarity index 61% copy from asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.130.query.sqlpp index 1168ba1de7..4ccbb8de34 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.130.query.sqlpp @@ -17,16 +17,10 @@ * under the License. */ -package org.apache.asterix.metadata.declared; +USE test; -import org.apache.asterix.om.types.ARecordType; -import org.apache.hyracks.algebricks.core.algebra.metadata.IWriteDataSink; -import org.apache.hyracks.api.exceptions.SourceLocation; - -public interface IExternalWriteDataSink extends IWriteDataSink { - ARecordType getItemType(); - - ARecordType getParquetSchema(); - - SourceLocation getSourceLoc(); -} +SELECT +joinDateTime as joinDateTime, get_type(joinDateTime) as joinDateTimeType, datetime(joinDateTime) as joinDateTimeCasted, get_type(datetime(joinDateTime)) as joinDateTimeCastedType, +joinDate as joinDate, get_type(joinDate) as joinDateType, date(joinDate) as joinDateCasted, get_type(date(joinDate)) as joinDateCastedType, +joinTime as joinTime, get_type(joinTime) as joinTimeType, time(joinTime) as joinTimeCasted, get_type(time(joinTime)) as joinTimeCastedType +FROM datetimeNotFormattedDataset; \ No newline at end of file diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.131.query.sqlpp similarity index 67% copy from asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.131.query.sqlpp index 1168ba1de7..ddd60e65ec 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.131.query.sqlpp @@ -17,16 +17,8 @@ * under the License. */ -package org.apache.asterix.metadata.declared; +USE test; -import org.apache.asterix.om.types.ARecordType; -import org.apache.hyracks.algebricks.core.algebra.metadata.IWriteDataSink; -import org.apache.hyracks.api.exceptions.SourceLocation; - -public interface IExternalWriteDataSink extends IWriteDataSink { - ARecordType getItemType(); - - ARecordType getParquetSchema(); - - SourceLocation getSourceLoc(); -} +SELECT +joinDateTime as joinDateTime, get_type(joinDateTime) as joinDateTimeType, parse_datetime(joinDateTime, "DD-MM-YYYYTss:mm:hh") as joinDateTimeCasted, get_type(parse_datetime(joinDateTime, "DD-MM-YYYYTss:mm:hh")) as joinDateTimeCastedType +FROM datetimeFormattedDataset; \ No newline at end of file diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.132.query.sqlpp similarity index 64% copy from asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.132.query.sqlpp index 1168ba1de7..ac6158ea19 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.132.query.sqlpp @@ -17,16 +17,9 @@ * under the License. */ -package org.apache.asterix.metadata.declared; +USE test; -import org.apache.asterix.om.types.ARecordType; -import org.apache.hyracks.algebricks.core.algebra.metadata.IWriteDataSink; -import org.apache.hyracks.api.exceptions.SourceLocation; - -public interface IExternalWriteDataSink extends IWriteDataSink { - ARecordType getItemType(); - - ARecordType getParquetSchema(); - - SourceLocation getSourceLoc(); -} +SELECT +joinDateTime as joinDateTime, get_type(joinDateTime) as joinDateTimeType, parse_datetime(joinDateTime, "DD-MM-YYYY") as joinDateTimeCasted, get_type(parse_datetime(joinDateTime, "DD-MM-YYYY")) as joinDateTimeCastedType, +joinDate as joinDate, get_type(joinDate) as joinDateType, parse_Date(joinDate, "DD-MM-YYYY") as joinDateCasted, get_type(parse_Date(joinDate, "DD-MM-YYYY")) as joinDateCastedType +FROM datetimeDateFormattedDataset; \ No newline at end of file diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.133.query.sqlpp similarity index 67% copy from asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.133.query.sqlpp index 1168ba1de7..870e83077f 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.133.query.sqlpp @@ -17,16 +17,8 @@ * under the License. */ -package org.apache.asterix.metadata.declared; +USE test; -import org.apache.asterix.om.types.ARecordType; -import org.apache.hyracks.algebricks.core.algebra.metadata.IWriteDataSink; -import org.apache.hyracks.api.exceptions.SourceLocation; - -public interface IExternalWriteDataSink extends IWriteDataSink { - ARecordType getItemType(); - - ARecordType getParquetSchema(); - - SourceLocation getSourceLoc(); -} +SELECT +joinTime as joinTime, get_type(joinTime) as joinTimeType, parse_time(joinTime, "hh:mm:ss.nnna") as joinTimeCasted, get_type(parse_time(joinTime, "hh:mm:ss.nnna")) as joinTimeCastedType +FROM datatimeTimeFormattedDataset; \ No newline at end of file diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.134.query.sqlpp similarity index 67% copy from asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.134.query.sqlpp index 1168ba1de7..332222d815 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.134.query.sqlpp @@ -17,16 +17,8 @@ * under the License. */ -package org.apache.asterix.metadata.declared; +USE test; -import org.apache.asterix.om.types.ARecordType; -import org.apache.hyracks.algebricks.core.algebra.metadata.IWriteDataSink; -import org.apache.hyracks.api.exceptions.SourceLocation; - -public interface IExternalWriteDataSink extends IWriteDataSink { - ARecordType getItemType(); - - ARecordType getParquetSchema(); - - SourceLocation getSourceLoc(); -} +SELECT +joinDateTime as joinDateTime, get_type(joinDateTime) as joinDateTimeType, parse_datetime(joinDateTime, "DD/MM/YYYY ss:mm:hh a") as joinDateTimeCasted, get_type(parse_datetime(joinDateTime, "DD/MM/YYYY ss:mm:hh a")) as joinDateTimeCastedType +FROM datetimeFormattedDataset2; \ No newline at end of file diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.135.query.sqlpp similarity index 67% copy from asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.135.query.sqlpp index 1168ba1de7..ffe5b55eec 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.135.query.sqlpp @@ -17,16 +17,8 @@ * under the License. */ -package org.apache.asterix.metadata.declared; +USE test; -import org.apache.asterix.om.types.ARecordType; -import org.apache.hyracks.algebricks.core.algebra.metadata.IWriteDataSink; -import org.apache.hyracks.api.exceptions.SourceLocation; - -public interface IExternalWriteDataSink extends IWriteDataSink { - ARecordType getItemType(); - - ARecordType getParquetSchema(); - - SourceLocation getSourceLoc(); -} +SELECT +joinDateTime as joinDateTime, get_type(joinDateTime) as joinDateTimeType, parse_datetime(joinDateTime, "DD/MMM/YYYY, ss:mm:hh a") as joinDateTimeCasted, get_type(parse_datetime(joinDateTime, "DD/MMM/YYYY, ss:mm:hh a")) as joinDateTimeCastedType +FROM datetimeFormattedWithCommaDataset; \ No newline at end of file diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.136.query.sqlpp similarity index 54% copy from asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.136.query.sqlpp index 1168ba1de7..aa3afadf03 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.136.query.sqlpp @@ -17,16 +17,10 @@ * under the License. */ -package org.apache.asterix.metadata.declared; +USE test; -import org.apache.asterix.om.types.ARecordType; -import org.apache.hyracks.algebricks.core.algebra.metadata.IWriteDataSink; -import org.apache.hyracks.api.exceptions.SourceLocation; - -public interface IExternalWriteDataSink extends IWriteDataSink { - ARecordType getItemType(); - - ARecordType getParquetSchema(); - - SourceLocation getSourceLoc(); -} +SELECT +joinDateTime as joinDateTime, get_type(joinDateTime) as joinDateTimeType, parse_datetime(joinDateTime, "DD/MMM/YYYY, ss:mm:hh a") as joinDateTimeCasted, get_type(parse_datetime(joinDateTime, "DD/MMM/YYYY, ss:mm:hh a")) as joinDateTimeCastedType, +joinDate as joinDate, get_type(joinDate) as joinDateType, parse_Date(joinDate, "DD/MMM/YYYY") as joinDateCasted, get_type(parse_Date(joinDate, "DD/MMM/YYYY")) as joinDateCastedType, +joinTime as joinTime, get_type(joinTime) as joinTimeType, parse_time(joinTime, "ss:mm:hh a") as joinTimeCasted, get_type(parse_time(joinTime, "ss:mm:hh a")) as joinTimeCastedType +FROM datetimeDateTimeFormattedWithCommaDataset; \ No newline at end of file diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.999.ddl.sqlpp similarity index 67% copy from asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.999.ddl.sqlpp index 1168ba1de7..220c7be7cb 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/csv/datetime/test.999.ddl.sqlpp @@ -17,16 +17,6 @@ * under the License. */ -package org.apache.asterix.metadata.declared; +DROP DATAVERSE test IF EXISTS; -import org.apache.asterix.om.types.ARecordType; -import org.apache.hyracks.algebricks.core.algebra.metadata.IWriteDataSink; -import org.apache.hyracks.api.exceptions.SourceLocation; -public interface IExternalWriteDataSink extends IWriteDataSink { - ARecordType getItemType(); - - ARecordType getParquetSchema(); - - SourceLocation getSourceLoc(); -} diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/csv/datetime/result.130.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/csv/datetime/result.130.adm new file mode 100644 index 0000000000..b6a231a60a --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/csv/datetime/result.130.adm @@ -0,0 +1 @@ +{ "joinDateTime": "2025-04-25T14:53:54.398", "joinDateTimeType": "string", "joinDateTimeCasted": datetime("2025-04-25T14:53:54.398"), "joinDateTimeCastedType": "datetime", "joinDate": "2025-04-25", "joinDateType": "string", "joinDateCasted": date("2025-04-25"), "joinDateCastedType": "date", "joinTime": "14:53:54.398", "joinTimeType": "string", "joinTimeCasted": time("14:53:54.398"), "joinTimeCastedType": "time" } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/csv/datetime/result.131.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/csv/datetime/result.131.adm new file mode 100644 index 0000000000..74c33ccc06 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/csv/datetime/result.131.adm @@ -0,0 +1 @@ +{ "joinDateTime": "25-04-2025T54:53:14", "joinDateTimeType": "string", "joinDateTimeCasted": datetime("2025-04-25T14:53:54.000"), "joinDateTimeCastedType": "datetime" } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/csv/datetime/result.132.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/csv/datetime/result.132.adm new file mode 100644 index 0000000000..1f221d1782 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/csv/datetime/result.132.adm @@ -0,0 +1 @@ +{ "joinDateTime": "25-04-2025", "joinDateTimeType": "string", "joinDateTimeCasted": datetime("2025-04-25T00:00:00.000"), "joinDateTimeCastedType": "datetime", "joinDate": "25-04-2025", "joinDateType": "string", "joinDateCasted": date("2025-04-25"), "joinDateCastedType": "date" } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/csv/datetime/result.133.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/csv/datetime/result.133.adm new file mode 100644 index 0000000000..2e3cb97fdf --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/csv/datetime/result.133.adm @@ -0,0 +1 @@ +{ "joinTime": "02:53:54.398PM", "joinTimeType": "string", "joinTimeCasted": time("14:53:54.398"), "joinTimeCastedType": "time" } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/csv/datetime/result.134.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/csv/datetime/result.134.adm new file mode 100644 index 0000000000..59d3024724 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/csv/datetime/result.134.adm @@ -0,0 +1 @@ +{ "joinDateTime": "25/04/2025 54:53:02 PM", "joinDateTimeType": "string", "joinDateTimeCasted": datetime("2025-04-25T14:53:54.000"), "joinDateTimeCastedType": "datetime" } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/csv/datetime/result.135.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/csv/datetime/result.135.adm new file mode 100644 index 0000000000..84e241a7eb --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/csv/datetime/result.135.adm @@ -0,0 +1 @@ +{ "joinDateTime": "25/APR/2025, 54:53:02 PM", "joinDateTimeType": "string", "joinDateTimeCasted": datetime("2025-04-25T14:53:54.000"), "joinDateTimeCastedType": "datetime" } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/csv/datetime/result.136.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/csv/datetime/result.136.adm new file mode 100644 index 0000000000..3d296435f2 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/csv/datetime/result.136.adm @@ -0,0 +1 @@ +{ "joinDateTime": "25/APR/2025, 54:53:02 PM", "joinDateTimeType": "string", "joinDateTimeCasted": datetime("2025-04-25T14:53:54.000"), "joinDateTimeCastedType": "datetime", "joinDate": "25/APR/2025", "joinDateType": "string", "joinDateCasted": date("2025-04-25"), "joinDateCastedType": "date", "joinTime": "54:53:02 PM", "joinTimeType": "string", "joinTimeCasted": time("14:53:54.000"), "joinTimeCastedType": "time" } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml index c9f9da661d..ae0a5207b2 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml @@ -365,6 +365,16 @@ <expected-error>'||' is not a valid escape. The length of a escape should be 1</expected-error> </compilation-unit> </test-case> + <test-case FilePath="copy-to/csv"> + <compilation-unit name="datetime"> + <placeholder name="adapter" value="S3" /> + <placeholder name="pathprefix" value="" /> + <placeholder name="path_prefix" value="" /> + <placeholder name="additionalProperties" value='"container":"playground",' /> + <placeholder name="additional_Properties" value='("container"="playground")' /> + <output-dir compare="Text">datetime</output-dir> + </compilation-unit> + </test-case> </test-group> <test-group name="aws-s3-external-dataset"> <test-case FilePath="external-dataset"> diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyToStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyToStatement.java index 5c89a9f371..ef76d14f25 100644 --- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyToStatement.java +++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyToStatement.java @@ -54,15 +54,16 @@ public class CopyToStatement extends AbstractStatement implements IReturningStat private List<Expression> partitionExpressions; private List<Expression> orderByList; private int varCounter; - private RecordTypeDefinition itemType; - private TypeExpression typeExpressionItemType; + private final RecordTypeDefinition itemType; + private final TypeExpression typeExpressionItemType; + private final Map<String, String> formatConfigs; public CopyToStatement(Namespace namespace, String datasetName, Query query, VariableExpr sourceVariable, ExternalDetailsDecl externalDetailsDecl, int varCounter, List<Expression> keyExpressions, boolean autogenerated) { this(namespace, datasetName, query, sourceVariable, externalDetailsDecl, new ArrayList<>(), new ArrayList<>(), new HashMap<>(), new ArrayList<>(), new ArrayList<>(), new ArrayList<>(), varCounter, keyExpressions, - autogenerated, null, null); + autogenerated, null, null, null); } public CopyToStatement(Namespace namespace, String datasetName, Query query, VariableExpr sourceVariable, @@ -72,7 +73,7 @@ public class CopyToStatement extends AbstractStatement implements IReturningStat List<OrderbyClause.NullOrderModifier> orderByNullModifierList, int varCounter) { this(namespace, datasetName, query, sourceVariable, externalDetailsDecl, pathExpressions, partitionExpressions, partitionsVariables, orderbyList, orderByModifiers, orderByNullModifierList, varCounter, - new ArrayList<>(), false, null, null); + new ArrayList<>(), false, null, null, null); } public CopyToStatement(Namespace namespace, String datasetName, Query query, VariableExpr sourceVariable, @@ -80,10 +81,10 @@ public class CopyToStatement extends AbstractStatement implements IReturningStat List<Expression> partitionExpressions, Map<Integer, VariableExpr> partitionsVariables, List<Expression> orderbyList, List<OrderbyClause.OrderModifier> orderByModifiers, List<OrderbyClause.NullOrderModifier> orderByNullModifierList, int varCounter, - TypeExpression typeExpressionItemType, RecordTypeDefinition itemType) { + TypeExpression typeExpressionItemType, RecordTypeDefinition itemType, Map<String, String> formatConfigs) { this(namespace, datasetName, query, sourceVariable, externalDetailsDecl, pathExpressions, partitionExpressions, partitionsVariables, orderbyList, orderByModifiers, orderByNullModifierList, varCounter, - new ArrayList<>(), false, typeExpressionItemType, itemType); + new ArrayList<>(), false, typeExpressionItemType, itemType, formatConfigs); } private CopyToStatement(Namespace namespace, String datasetName, Query query, VariableExpr sourceVariable, @@ -92,7 +93,7 @@ public class CopyToStatement extends AbstractStatement implements IReturningStat List<Expression> orderbyList, List<OrderbyClause.OrderModifier> orderByModifiers, List<OrderbyClause.NullOrderModifier> orderByNullModifierList, int varCounter, List<Expression> keyExpressions, boolean autogenerated, TypeExpression typeExpressionItemType, - RecordTypeDefinition itemType) { + RecordTypeDefinition itemType, Map<String, String> formatConfigs) { this.namespace = namespace; this.datasetName = datasetName; this.query = query; @@ -109,6 +110,7 @@ public class CopyToStatement extends AbstractStatement implements IReturningStat this.autogenerated = autogenerated; this.itemType = itemType; this.typeExpressionItemType = typeExpressionItemType; + this.formatConfigs = formatConfigs; if (pathExpressions.isEmpty()) { // Ensure path expressions to have at least an empty string @@ -218,6 +220,10 @@ public class CopyToStatement extends AbstractStatement implements IReturningStat return typeExpressionItemType; } + public Map<String, String> getFormatConfigs() { + return formatConfigs; + } + @Override public int getVarCounter() { return varCounter; diff --git a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj index 7a46e95c9b..0c95c77ee3 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj +++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj @@ -2978,6 +2978,8 @@ CopyToStatement CopyToStatement(Token startToken, Pair<Namespace, Identifier> na RecordTypeDefinition recordTypeDefinition = null; TypeExpression typeExpr = null; Boolean isRecordTypeDefinition = false; + Map<String, String> formatConfigs = null; + String propertyName = null, propertyValue = null; List<Expression> partitionExprs = new ArrayList<Expression>(); Map<Integer, VariableExpr> partitionVarExprs = new HashMap<Integer, VariableExpr>(); @@ -2994,7 +2996,8 @@ CopyToStatement CopyToStatement(Token startToken, Pair<Namespace, Identifier> na recordTypeDefinition = RecordTypeDef(); isRecordTypeDefinition = true; } - <RIGHTPAREN>) ? + <RIGHTPAREN> + )? (<AS> { if (isRecordTypeDefinition == false) { @@ -3004,6 +3007,14 @@ CopyToStatement CopyToStatement(Token startToken, Pair<Namespace, Identifier> na } } )? + (LOOKAHEAD(2) <IDENTIFIER> { propertyName = token.image.toLowerCase(); } propertyValue = StringLiteral() + { + if (formatConfigs == null) { + formatConfigs = new HashMap<String, String>(); + } + formatConfigs.put(propertyName, propertyValue); + } + )* <WITH> withRecord = RecordConstructor() { ExternalDetailsDecl edd = new ExternalDetailsDecl(); @@ -3018,7 +3029,7 @@ CopyToStatement CopyToStatement(Token startToken, Pair<Namespace, Identifier> na usedAlias = new VariableExpr(SqlppVariableUtil.toInternalVariableIdentifier(datasetName)); } - CopyToStatement stmt = new CopyToStatement(namespace, datasetName, query, usedAlias, edd, pathExprs, partitionExprs, partitionVarExprs, orderbyList, orderbyModifierList, orderbyNullModifierList, getVarCounter(), typeExpr, recordTypeDefinition); + CopyToStatement stmt = new CopyToStatement(namespace, datasetName, query, usedAlias, edd, pathExprs, partitionExprs, partitionVarExprs, orderbyList, orderbyModifierList, orderbyNullModifierList, getVarCounter(), typeExpr, recordTypeDefinition, formatConfigs); return addSourceLocation(stmt, startToken); } } diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java index 1168ba1de7..157f0f84cf 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IExternalWriteDataSink.java @@ -19,6 +19,8 @@ package org.apache.asterix.metadata.declared; +import java.util.Map; + import org.apache.asterix.om.types.ARecordType; import org.apache.hyracks.algebricks.core.algebra.metadata.IWriteDataSink; import org.apache.hyracks.api.exceptions.SourceLocation; @@ -28,5 +30,7 @@ public interface IExternalWriteDataSink extends IWriteDataSink { ARecordType getParquetSchema(); + Map<String, String> getFormatConfigs(); + SourceLocation getSourceLoc(); } diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/WriteDataSink.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/WriteDataSink.java index 4a10f7fcaf..4541162e1e 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/WriteDataSink.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/WriteDataSink.java @@ -29,15 +29,17 @@ public class WriteDataSink implements IExternalWriteDataSink { private final String adapterName; private final Map<String, String> configuration; private final ARecordType itemType; + private final Map<String, String> formatConfigs; private final ARecordType parquetSchema; private final SourceLocation sourceLoc; public WriteDataSink(String adapterName, Map<String, String> configuration, ARecordType itemType, - ARecordType parquetSchema, SourceLocation sourceLoc) { + ARecordType parquetSchema, Map<String, String> formatConfigs, SourceLocation sourceLoc) { this.adapterName = adapterName; this.configuration = configuration; this.itemType = itemType; this.parquetSchema = parquetSchema; + this.formatConfigs = formatConfigs; this.sourceLoc = sourceLoc; } @@ -46,6 +48,7 @@ public class WriteDataSink implements IExternalWriteDataSink { this.configuration = new HashMap<>(writeDataSink.configuration); this.itemType = writeDataSink.itemType; this.parquetSchema = writeDataSink.parquetSchema; + this.formatConfigs = writeDataSink.getFormatConfigs(); this.sourceLoc = writeDataSink.sourceLoc; } @@ -74,6 +77,10 @@ public class WriteDataSink implements IExternalWriteDataSink { return configuration; } + public Map<String, String> getFormatConfigs() { + return formatConfigs; + } + @Override public IWriteDataSink createCopy() { return new WriteDataSink(this); diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java index 23235c5328..2f9af01e61 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java @@ -183,9 +183,10 @@ public class ExternalWriterProvider { compressStreamFactory = createCompressionStreamFactory(appCtx, compression, configuration); if (sink instanceof IExternalWriteDataSink externalSink) { ARecordType itemType = externalSink.getItemType(); + Map<String, String> formatConfigs = externalSink.getFormatConfigs(); if (itemType != null) { printerFactory = CSVPrinterFactoryProvider - .createInstance(itemType, externalSink.getConfiguration(), externalSink.getSourceLoc()) + .createInstance(itemType, externalSink.getConfiguration(), formatConfigs, externalSink.getSourceLoc()) .getPrinterFactory(sourceType); externalPrinterFactory = new CsvExternalFilePrinterFactory(printerFactory, compressStreamFactory); diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/PrintTools.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/PrintTools.java index 372b5fd6f3..92fb1bb359 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/PrintTools.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/PrintTools.java @@ -49,7 +49,7 @@ public class PrintTools { private static final long CHRONON_OF_DAY = TimeUnit.DAYS.toMillis(1); public static void printDateString(byte[] b, int s, int l, PrintStream ps) throws HyracksDataException { - long chrononTime = AInt32SerializerDeserializer.getInt(b, s + 1) * CHRONON_OF_DAY; + long chrononTime = getDateChronon(b, s + 1); try { gCalInstance.getExtendStringRepUntilField(chrononTime, ps, GregorianCalendarSystem.Fields.YEAR, @@ -59,8 +59,12 @@ public class PrintTools { } } + public static long getDateChronon(byte[] b, int s) { + return AInt32SerializerDeserializer.getInt(b, s) * CHRONON_OF_DAY; + } + public static void printDateTimeString(byte[] b, int s, int l, PrintStream ps) throws HyracksDataException { - long chrononTime = AInt64SerializerDeserializer.getLong(b, s + 1); + long chrononTime = getDateTimeChronon(b, s + 1); try { gCalInstance.getExtendStringRepUntilField(chrononTime, ps, GregorianCalendarSystem.Fields.YEAR, @@ -70,9 +74,13 @@ public class PrintTools { } } + public static long getDateTimeChronon(byte[] b, int s) { + return AInt64SerializerDeserializer.getLong(b, s); + } + public static void printDayTimeDurationString(byte[] b, int s, int l, PrintStream ps) throws HyracksDataException { boolean positive = true; - long milliseconds = AInt64SerializerDeserializer.getLong(b, s + 1); + long milliseconds = getDateTimeChronon(b, s + 1); // set the negative flag. "||" is necessary in case that months field is not there (so it is 0) if (milliseconds < 0) { @@ -218,7 +226,7 @@ public class PrintTools { } public static void printTimeString(byte[] b, int s, int l, PrintStream ps) throws HyracksDataException { - int time = AInt32SerializerDeserializer.getInt(b, s + 1); + int time = getTimeChronon(b, s + 1); try { gCalInstance.getExtendStringRepUntilField(time, ps, GregorianCalendarSystem.Fields.HOUR, @@ -228,6 +236,10 @@ public class PrintTools { } } + public static int getTimeChronon(byte[] b, int s) { + return AInt32SerializerDeserializer.getInt(b, s); + } + public static void printDoubleForJson(byte[] b, int s, PrintStream ps) { final double d = ADoubleSerializerDeserializer.getDouble(b, s + 1); if (Double.isFinite(d)) { diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/ADatePrinterFactory.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/ADatePrinterFactory.java index dd03c970f5..83146ecef8 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/ADatePrinterFactory.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/ADatePrinterFactory.java @@ -18,8 +18,6 @@ */ package org.apache.asterix.dataflow.data.nontagged.printers.csv; -import java.io.PrintStream; - import org.apache.asterix.dataflow.data.nontagged.printers.PrintTools; import org.apache.hyracks.algebricks.data.IPrinter; import org.apache.hyracks.algebricks.data.IPrinterFactory; @@ -30,11 +28,7 @@ public class ADatePrinterFactory implements IPrinterFactory { private static final long serialVersionUID = 1L; public static final ADatePrinterFactory INSTANCE = new ADatePrinterFactory(); - public static final IPrinter PRINTER = (byte[] b, int s, int l, PrintStream ps) -> { - ps.print("\""); - PrintTools.printDateString(b, s, l, ps); - ps.print("\""); - }; + public static final IPrinter PRINTER = PrintTools::printDateString; @Override public IPrinter createPrinter(IEvaluatorContext context) { diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/ADateTimePrinterFactory.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/ADateTimePrinterFactory.java index b5b6185a2b..f28c012886 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/ADateTimePrinterFactory.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/ADateTimePrinterFactory.java @@ -18,8 +18,6 @@ */ package org.apache.asterix.dataflow.data.nontagged.printers.csv; -import java.io.PrintStream; - import org.apache.asterix.dataflow.data.nontagged.printers.PrintTools; import org.apache.hyracks.algebricks.data.IPrinter; import org.apache.hyracks.algebricks.data.IPrinterFactory; @@ -30,11 +28,7 @@ public class ADateTimePrinterFactory implements IPrinterFactory { private static final long serialVersionUID = 1L; public static final ADateTimePrinterFactory INSTANCE = new ADateTimePrinterFactory(); - public static final IPrinter PRINTER = (byte[] b, int s, int l, PrintStream ps) -> { - ps.print("\""); - PrintTools.printDateTimeString(b, s, l, ps); - ps.print("\""); - }; + public static final IPrinter PRINTER = PrintTools::printDateTimeString; @Override public IPrinter createPrinter(IEvaluatorContext context) { diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/AObjectPrinterFactory.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/AObjectPrinterFactory.java index 2f4d18024f..c3ef619c7f 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/AObjectPrinterFactory.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/AObjectPrinterFactory.java @@ -26,10 +26,15 @@ import static org.apache.asterix.common.utils.CSVConstants.KEY_NULL_STR; import static org.apache.asterix.common.utils.CSVConstants.KEY_QUOTE; import static org.apache.asterix.dataflow.data.nontagged.printers.csv.CSVUtils.DEFAULT_VALUES; import static org.apache.asterix.dataflow.data.nontagged.printers.csv.CSVUtils.getCharOrDefault; +import static org.apache.asterix.om.base.temporal.DateTimeFormatUtils.DateTimeParseMode.DATETIME; +import static org.apache.asterix.om.base.temporal.DateTimeFormatUtils.DateTimeParseMode.DATE_ONLY; +import static org.apache.asterix.om.base.temporal.DateTimeFormatUtils.DateTimeParseMode.TIME_ONLY; import java.io.PrintStream; import java.util.Map; +import org.apache.asterix.dataflow.data.nontagged.printers.PrintTools; +import org.apache.asterix.om.base.temporal.DateTimeFormatUtils; import org.apache.asterix.om.pointables.ARecordVisitablePointable; import org.apache.asterix.om.pointables.base.DefaultOpenFieldType; import org.apache.asterix.om.pointables.printer.IPrintVisitor; @@ -42,12 +47,17 @@ import org.apache.hyracks.algebricks.data.IPrinter; import org.apache.hyracks.algebricks.data.IPrinterFactory; import org.apache.hyracks.api.context.IEvaluatorContext; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.data.std.primitive.UTF8StringPointable; public class AObjectPrinterFactory implements IPrinterFactory { private static final long serialVersionUID = 1L; + private static final String DATE_FORMAT_PROPERTY_NAME = "date"; + private static final String TIME_FORMAT_PROPERTY_NAME = "time"; + private static final String DATETIME_FORMAT_PROPERTY_NAME = "datetime"; private static final String DEFAULT_NULL_STRING = ""; private final ARecordType itemType; private final Map<String, String> configuration; + private final Map<String, String> formatConfigs; private final boolean emptyFieldAsNull; private final String nullString; private final char quote; @@ -55,9 +65,28 @@ public class AObjectPrinterFactory implements IPrinterFactory { private final char escape; private final char delimiter; - private AObjectPrinterFactory(ARecordType itemType, Map<String, String> configuration) { + private DateTimeFormatUtils dateTimeFormatUtils; + private String timeFormat; + private String dateFormat; + private String datetimeFormat; + private byte[] timeFormatBytes; + private byte[] dateFormatBytes; + private byte[] datetimeFormatBytes; + private int timeFormatOffset; + private int dateFormatOffset; + private int datetimeFormatOffset; + private int timeFormatLength; + private int dateFormatLength; + private int datetimeFormatLength; + private boolean quoteTime; + private boolean quoteDate; + private boolean quoteDatetime; + + private AObjectPrinterFactory(ARecordType itemType, Map<String, String> formatConfigs, + Map<String, String> configuration) { this.itemType = itemType; this.configuration = configuration; + this.formatConfigs = formatConfigs; this.emptyFieldAsNull = Boolean.parseBoolean(configuration.get(KEY_EMPTY_STRING_AS_NULL)); this.nullString = configuration.get(KEY_NULL_STR) != null ? configuration.get(KEY_NULL_STR) : DEFAULT_NULL_STRING; @@ -65,10 +94,12 @@ public class AObjectPrinterFactory implements IPrinterFactory { this.quote = getCharOrDefault(configuration.get(KEY_QUOTE), DEFAULT_VALUES.get(KEY_QUOTE)); this.escape = getCharOrDefault(configuration.get(KEY_ESCAPE), DEFAULT_VALUES.get(KEY_ESCAPE)); this.delimiter = getCharOrDefault(configuration.get(KEY_DELIMITER), DEFAULT_VALUES.get(KEY_DELIMITER)); + extractDateTimeFormats(formatConfigs); } - public static AObjectPrinterFactory createInstance(ARecordType itemType, Map<String, String> configuration) { - return new AObjectPrinterFactory(itemType, configuration); + public static AObjectPrinterFactory createInstance(ARecordType itemType, Map<String, String> formatConfigs, + Map<String, String> configuration) { + return new AObjectPrinterFactory(itemType, formatConfigs, configuration); } public boolean printFlatValue(ATypeTag typeTag, byte[] b, int s, int l, PrintStream ps) @@ -100,13 +131,31 @@ public class AObjectPrinterFactory implements IPrinterFactory { ADoublePrinterFactory.PRINTER.print(b, s, l, ps); return true; case DATE: - ADatePrinterFactory.PRINTER.print(b, s, l, ps); + if (dateFormat == null) { + ADatePrinterFactory.PRINTER.print(b, s, l, ps); + } else { + long chronon = PrintTools.getDateChronon(b, s + 1); + printFormattedDatetime(dateFormatBytes, dateFormatOffset, dateFormatLength, ps, chronon, DATE_ONLY, + quoteDate); + } return true; case TIME: - ATimePrinterFactory.PRINTER.print(b, s, l, ps); + if (timeFormat == null) { + ATimePrinterFactory.PRINTER.print(b, s, l, ps); + } else { + long chronon = PrintTools.getTimeChronon(b, s + 1); + printFormattedDatetime(timeFormatBytes, timeFormatOffset, timeFormatLength, ps, chronon, TIME_ONLY, + quoteTime); + } return true; case DATETIME: - ADateTimePrinterFactory.PRINTER.print(b, s, l, ps); + if (datetimeFormat == null) { + ADateTimePrinterFactory.PRINTER.print(b, s, l, ps); + } else { + long chronon = PrintTools.getDateTimeChronon(b, s + 1); + printFormattedDatetime(datetimeFormatBytes, datetimeFormatOffset, datetimeFormatLength, ps, chronon, + DATETIME, quoteDatetime); + } return true; case DURATION: ADurationPrinterFactory.PRINTER.print(b, s, l, ps); @@ -161,7 +210,7 @@ public class AObjectPrinterFactory implements IPrinterFactory { final ARecordVisitablePointable recordVisitablePointable = new ARecordVisitablePointable(DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE); final Pair<PrintStream, ATypeTag> streamTag = new Pair<>(null, null); - final IPrintVisitor visitor = new APrintVisitor(context, itemType, configuration); + final IPrintVisitor visitor = new APrintVisitor(context, itemType, formatConfigs, configuration); return (byte[] b, int s, int l, PrintStream ps) -> { ATypeTag typeTag = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(b[s]); @@ -185,4 +234,50 @@ public class AObjectPrinterFactory implements IPrinterFactory { private void printString(byte[] b, int s, int l, PrintStream ps) throws HyracksDataException { CSVUtils.printString(b, s, l, ps, quote, forceQuote, escape, delimiter); } + + private void printFormattedDatetime(byte[] formatBytes, int formatOffset, int formatLength, PrintStream ps, + long chronon, DateTimeFormatUtils.DateTimeParseMode dateTimeParseMode, boolean shouldQuote) + throws HyracksDataException { + if (shouldQuote) { + ps.print("\""); + } + dateTimeFormatUtils.printDateTime(chronon, formatBytes, formatOffset, formatLength, ps, dateTimeParseMode); + if (shouldQuote) { + ps.print("\""); + } + } + + private void extractDateTimeFormats(Map<String, String> formatConfigs) { + if (formatConfigs != null && !formatConfigs.isEmpty()) { + timeFormat = formatConfigs.get(TIME_FORMAT_PROPERTY_NAME); + dateFormat = formatConfigs.get(DATE_FORMAT_PROPERTY_NAME); + datetimeFormat = formatConfigs.get(DATETIME_FORMAT_PROPERTY_NAME); + dateTimeFormatUtils = DateTimeFormatUtils.getInstance(); + + UTF8StringPointable pointable; + if (timeFormat != null) { + pointable = UTF8StringPointable.generateUTF8Pointable(timeFormat); + timeFormatBytes = pointable.getByteArray(); + timeFormatOffset = pointable.getStartOffset() + 1; + timeFormatLength = pointable.getLength() - 1; + quoteTime = timeFormat.indexOf(this.delimiter) != -1; + } + + if (dateFormat != null) { + pointable = UTF8StringPointable.generateUTF8Pointable(dateFormat); + dateFormatBytes = pointable.getByteArray(); + dateFormatOffset = pointable.getStartOffset() + 1; + dateFormatLength = pointable.getLength() - 1; + quoteDate = dateFormat.indexOf(this.delimiter) != -1; + } + + if (datetimeFormat != null) { + pointable = UTF8StringPointable.generateUTF8Pointable(datetimeFormat); + datetimeFormatBytes = pointable.getByteArray(); + datetimeFormatOffset = pointable.getStartOffset() + 1; + datetimeFormatLength = pointable.getLength() - 1; + quoteDatetime = datetimeFormat.indexOf(this.delimiter) != -1; + } + } + } } diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/ARecordPrinterFactory.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/ARecordPrinterFactory.java index d3ec3883ea..cef0e6e3f0 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/ARecordPrinterFactory.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/ARecordPrinterFactory.java @@ -39,11 +39,14 @@ public class ARecordPrinterFactory implements IPrinterFactory { private static final long serialVersionUID = 1L; private final ARecordType recType; private final ARecordType itemType; + private final Map<String, String> formatConfigs; private final Map<String, String> configuration; - public ARecordPrinterFactory(ARecordType recType, ARecordType itemType, Map<String, String> configuration) { + public ARecordPrinterFactory(ARecordType recType, ARecordType itemType, Map<String, String> formatConfigs, + Map<String, String> configuration) { this.recType = recType; this.itemType = itemType; + this.formatConfigs = formatConfigs; this.configuration = configuration; } @@ -54,7 +57,7 @@ public class ARecordPrinterFactory implements IPrinterFactory { final IAType inputType = recType == null ? DefaultOpenFieldType.getDefaultOpenFieldType(ATypeTag.OBJECT) : recType; final IVisitablePointable recAccessor = allocator.allocateRecordValue(inputType); - final APrintVisitor printVisitor = new APrintVisitor(context, itemType, configuration); + final APrintVisitor printVisitor = new APrintVisitor(context, itemType, formatConfigs, configuration); final Pair<PrintStream, ATypeTag> arg = new Pair<>(null, null); return new IPrinter() { diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/ATimePrinterFactory.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/ATimePrinterFactory.java index 2a73c56fb6..e647ec8031 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/ATimePrinterFactory.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/ATimePrinterFactory.java @@ -18,8 +18,6 @@ */ package org.apache.asterix.dataflow.data.nontagged.printers.csv; -import java.io.PrintStream; - import org.apache.asterix.dataflow.data.nontagged.printers.PrintTools; import org.apache.hyracks.algebricks.data.IPrinter; import org.apache.hyracks.algebricks.data.IPrinterFactory; @@ -30,11 +28,7 @@ public class ATimePrinterFactory implements IPrinterFactory { private static final long serialVersionUID = 1L; public static final ATimePrinterFactory INSTANCE = new ATimePrinterFactory(); - public static final IPrinter PRINTER = (byte[] b, int s, int l, PrintStream ps) -> { - ps.print("\""); - PrintTools.printTimeString(b, s, l, ps); - ps.print("\""); - }; + public static final IPrinter PRINTER = PrintTools::printTimeString; @Override public IPrinter createPrinter(IEvaluatorContext context) { diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/CSVPrinterFactoryProvider.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/CSVPrinterFactoryProvider.java index 4d28ce4d1c..799df8dc10 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/CSVPrinterFactoryProvider.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/CSVPrinterFactoryProvider.java @@ -66,20 +66,22 @@ import org.apache.hyracks.api.exceptions.SourceLocation; public class CSVPrinterFactoryProvider implements IPrinterFactoryProvider { private final ARecordType itemType; private final Map<String, String> configuration; + private final Map<String, String> formatConfigs; private final SourceLocation sourceLocation; public static final CSVPrinterFactoryProvider INSTANCE = - new CSVPrinterFactoryProvider(null, Collections.emptyMap(), null); + new CSVPrinterFactoryProvider(null, Collections.emptyMap(), null, null); public static CSVPrinterFactoryProvider createInstance(ARecordType itemType, Map<String, String> configuration, - SourceLocation sourceLocation) { - return new CSVPrinterFactoryProvider(itemType, configuration, sourceLocation); + Map<String, String> formatConfigs, SourceLocation sourceLocation) { + return new CSVPrinterFactoryProvider(itemType, configuration, formatConfigs, sourceLocation); } private CSVPrinterFactoryProvider(ARecordType itemType, Map<String, String> configuration, - SourceLocation sourceLocation) { + Map<String, String> formatConfigs, SourceLocation sourceLocation) { this.itemType = itemType; this.configuration = configuration; + this.formatConfigs = formatConfigs; this.sourceLocation = sourceLocation; } @@ -137,7 +139,7 @@ public class CSVPrinterFactoryProvider implements IPrinterFactoryProvider { configuration.get(KEY_FORCE_QUOTE), configuration.get(KEY_ESCAPE), configuration.get(KEY_DELIMITER)); case OBJECT: - return new ARecordPrinterFactory((ARecordType) type, itemType, configuration); + return new ARecordPrinterFactory((ARecordType) type, itemType, formatConfigs, configuration); case ARRAY: throw new NotImplementedException("'OrderedList' type unsupported for CSV output"); case MULTISET: @@ -167,7 +169,7 @@ public class CSVPrinterFactoryProvider implements IPrinterFactoryProvider { break; } } - return AObjectPrinterFactory.createInstance(itemType, configuration); + return AObjectPrinterFactory.createInstance(itemType, formatConfigs, configuration); } } diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/printer/csv/APrintVisitor.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/printer/csv/APrintVisitor.java index 7eadca5d4b..7d5a52df86 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/printer/csv/APrintVisitor.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/printer/csv/APrintVisitor.java @@ -42,13 +42,16 @@ import org.apache.hyracks.api.exceptions.HyracksDataException; public class APrintVisitor extends AbstractPrintVisitor { private final IEvaluatorContext context; private final ARecordType itemType; + private final Map<String, String> formatConfigs; private final Map<String, String> configuration; private AObjectPrinterFactory objectPrinterFactory; - public APrintVisitor(IEvaluatorContext context, ARecordType itemType, Map<String, String> configuration) { + public APrintVisitor(IEvaluatorContext context, ARecordType itemType, Map<String, String> formatConfigs, + Map<String, String> configuration) { super(); this.context = context; this.itemType = itemType; + this.formatConfigs = formatConfigs; this.configuration = configuration; } @@ -69,7 +72,7 @@ public class APrintVisitor extends AbstractPrintVisitor { protected boolean printFlatValue(ATypeTag typeTag, byte[] b, int s, int l, PrintStream ps) throws HyracksDataException { if (objectPrinterFactory == null) { - objectPrinterFactory = AObjectPrinterFactory.createInstance(itemType, configuration); + objectPrinterFactory = AObjectPrinterFactory.createInstance(itemType, formatConfigs, configuration); } return objectPrinterFactory.printFlatValue(typeTag, b, s, l, ps); }
