>From Hussain Towaileb <[email protected]>: Hussain Towaileb has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21303?usp=email )
Change subject: [ASTERIXDB-3634][EXT]: default datetime types to return datetime ...................................................................... [ASTERIXDB-3634][EXT]: default datetime types to return datetime Details: set the following properties to default to "false": - date-to-int - time-to-int - timestamp-to-long Ext-ref: MB-72169 Change-Id: I3073631ac331fa14749048f6fc143b753ac1db2d Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21303 Tested-by: Jenkins <[email protected]> Reviewed-by: Hussain Towaileb <[email protected]> Reviewed-by: Murtadha Hubail <[email protected]> --- M asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/iceberg/IcebergTest.java M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/all-data-types/test.010.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/all-data-types/test.040.query.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/all-data-types/test.999.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/results/iceberg/all-data-types/result.040.adm M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/iceberg/converter/IcebergConverterContext.java 7 files changed, 82 insertions(+), 51 deletions(-) Approvals: Anon. E. Moose #1000171: Hussain Towaileb: Looks good to me, but someone else must approve Murtadha Hubail: Looks good to me, approved Jenkins: Verified diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java index e40e931..b9b0858 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java @@ -55,7 +55,6 @@ public class LocalCloudUtilAdobeMock { private static final Logger LOGGER = LogManager.getLogger(); - private static final String S3MOCK_CONTAINER_NAME = "adobe-s3-mock"; public static final String CLOUD_STORAGE_BUCKET = "cloud-storage-container"; public static final String S3MOCK_VERSION_TAG = "4.7.0"; @@ -90,8 +89,7 @@ shutdownSilently(); // Starting S3 mock server to be used instead of real S3 server LOGGER.info("Starting S3 mock server"); - s3Mock = new S3MockContainer(S3MOCK_VERSION_TAG).withRetainFilesOnExit(!cleanStart) - .withCreateContainerCmdModifier(cmd -> cmd.withName(S3MOCK_CONTAINER_NAME)); + s3Mock = new S3MockContainer(S3MOCK_VERSION_TAG).withRetainFilesOnExit(!cleanStart); if (!cleanStart) { Path s3MockDataDir = Path.of("target", "s3mock"); boolean existingData = s3MockDataDir.toFile().exists(); diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/iceberg/IcebergTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/iceberg/IcebergTest.java index a2a5b84..c2b731b 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/iceberg/IcebergTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/iceberg/IcebergTest.java @@ -127,7 +127,6 @@ private static S3MockContainer s3Mock; // Nessie config - private static final String NESSIE_CONTAINER_NAME = "iceberg-nessie"; private static GenericContainer<?> nessie; private static final DockerImageName NESSIE_IMAGE = DockerImageName.parse("ghcr.io/projectnessie/nessie:0.107.5"); private static final int NESSIE_PORT = 19120; @@ -148,8 +147,7 @@ final TestExecutor testExecutor = new TestExecutor(); LOGGER.info("Starting S3 mock and Nessie containers"); s3Mock = LocalCloudUtilAdobeMock.startS3CloudEnvironment(true); - nessie = new GenericContainer<>(NESSIE_IMAGE).withExposedPorts(NESSIE_PORT) - .withCreateContainerCmdModifier(cmd -> cmd.withName(NESSIE_CONTAINER_NAME)); + nessie = new GenericContainer<>(NESSIE_IMAGE).withExposedPorts(NESSIE_PORT); nessie.setPortBindings(List.of(NESSIE_PORT + ":" + NESSIE_PORT)); nessie.start(); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/all-data-types/test.010.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/all-data-types/test.010.ddl.sqlpp index b434ebb..ba010cd 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/all-data-types/test.010.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/all-data-types/test.010.ddl.sqlpp @@ -57,4 +57,19 @@ ("date-to-int"="false"), ("time-to-int"="false"), ("timestamp-to-long"="false") +); + +CREATE TYPE allTypesTimesDefaultExpectingNotConverted AS open {}; +CREATE EXTERNAL COLLECTION allTypesTimesDefaultExpectingNotConverted(allTypesTimesDefaultExpectingNotConverted) +USING S3 +( + ("table-format"="iceberg"), + ("namespace"="my_namespace"), + ("tableName"="allTypes"), + ("region"="custom-region"), + ("serviceEndpoint"="%DOCKER_S3_CONTAINER%"), + ("pathStyleAddressing"="true"), + ("catalogName"="myNessieCatalog"), + ("format"="parquet"), + ("decimal-to-double"="true") ); \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/all-data-types/test.040.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/all-data-types/test.040.query.sqlpp new file mode 100644 index 0000000..c834ec1 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/all-data-types/test.040.query.sqlpp @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +SELECT + bool_field, + byte_field, + short_field, + int_field, + long_field, + float_field, + double_field, + decimal_field, + string_field, + varchar_field, + char_field, + uuid_field, + print_binary(binary_field, "base64") as binary_field, + print_binary(fixed_field, "base64") as fixed_field, + date_field, + time_field, + timestamp_field, + timestamp_ntz_field, + timestamp_nano_field, + interval_ym_field, + interval_dt_field, + print_binary(geometry_field, "base64") as geometry_field, + print_binary(geography_field, "base64") as geography_field, + struct_field, + list_field, + map_field, + variant_field, + unknown_field +FROM allTypesTimesDefaultExpectingNotConverted; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/all-data-types/test.999.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/all-data-types/test.999.ddl.sqlpp index 6fc7675..6fa5c73 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/all-data-types/test.999.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/all-data-types/test.999.ddl.sqlpp @@ -19,6 +19,8 @@ DROP COLLECTION allTypesTimesConverted; DROP COLLECTION allTypesTimesNotConverted; +DROP COLLECTION allTypesTimesDefaultExpectingNotConverted; DROP TYPE allTypesTimesConverted; DROP TYPE allTypesTimesNotConverted; +DROP TYPE allTypesTimesDefaultExpectingNotConverted; DROP CATALOG myNessieCatalog; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/iceberg/all-data-types/result.040.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/iceberg/all-data-types/result.040.adm new file mode 100644 index 0000000..4dfcb3a --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/iceberg/all-data-types/result.040.adm @@ -0,0 +1 @@ +{ "binary_field": "AQIDBAUGBwg=", "fixed_field": "SGVsbG8gV29ybGQ=", "geometry_field": "AQIDBAUGBwgJCg==", "geography_field": "AQIDBAUGBwgJCg0=", "bool_field": true, "byte_field": 42, "short_field": 1000, "int_field": 42, "long_field": 9223372036854775807, "float_field": 3.14, "double_field": 2.718281828459045, "decimal_field": 12345.6789, "string_field": "hello world", "varchar_field": "varchar value one", "char_field": "Hi", "uuid_field": uuid("550e8400-e29b-41d4-a716-446655440000"), "date_field": date("2024-01-01"), "time_field": time("10:20:30.000"), "timestamp_field": datetime("2024-02-03T22:40:00.000"), "timestamp_ntz_field": datetime("2024-02-04T12:00:00.000"), "timestamp_nano_field": datetime("2024-02-03T22:40:00.000"), "interval_ym_field": 14, "interval_dt_field": 37230000000, "struct_field": { "name": "Alice", "age": 30, "active": true }, "list_field": [ "a", "b", "c" ], "map_field": { "key1": "value1", "key2": "100" }, "variant_field": "string value", "unknown_field": null } \ No newline at end of file diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/iceberg/converter/IcebergConverterContext.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/iceberg/converter/IcebergConverterContext.java index c778b69..c7713ac 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/iceberg/converter/IcebergConverterContext.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/iceberg/converter/IcebergConverterContext.java @@ -18,7 +18,12 @@ */ package org.apache.asterix.external.input.record.reader.aws.iceberg.converter; -import java.io.DataOutput; +import static org.apache.asterix.external.util.ExternalDataConstants.FALSE; +import static org.apache.asterix.external.util.ExternalDataConstants.IcebergOptions.DATE_AS_INT; +import static org.apache.asterix.external.util.ExternalDataConstants.IcebergOptions.DECIMAL_TO_DOUBLE; +import static org.apache.asterix.external.util.ExternalDataConstants.IcebergOptions.TIMESTAMP_AS_LONG; +import static org.apache.asterix.external.util.ExternalDataConstants.IcebergOptions.TIME_AS_INT; + import java.time.ZoneId; import java.time.ZoneOffset; import java.util.List; @@ -27,43 +32,24 @@ import org.apache.asterix.external.parser.jackson.ParserContext; import org.apache.asterix.external.util.ExternalDataConstants; -import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider; -import org.apache.asterix.om.base.ADate; -import org.apache.asterix.om.base.ADateTime; -import org.apache.asterix.om.base.AMutableDate; -import org.apache.asterix.om.base.AMutableDateTime; -import org.apache.asterix.om.types.BuiltinType; -import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer; -import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.exceptions.Warning; public class IcebergConverterContext extends ParserContext { - @SuppressWarnings("unchecked") - private final ISerializerDeserializer<ADate> dateSerDer = - SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ADATE); - @SuppressWarnings("unchecked") - private final ISerializerDeserializer<ADateTime> datetimeSerDer = - SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ADATETIME); + private final boolean decimalToDouble; - private final boolean timestampAsLong; private final boolean dateAsInt; private final boolean timeAsInt; - + private final boolean timestampAsLong; private final ZoneId timeZoneId; - private final AMutableDate mutableDate = new AMutableDate(0); - private final AMutableDateTime mutableDateTime = new AMutableDateTime(0); private final List<Warning> warnings; public IcebergConverterContext(Map<String, String> configuration, List<Warning> warnings) { this.warnings = warnings; - decimalToDouble = Boolean.parseBoolean(configuration - .getOrDefault(ExternalDataConstants.IcebergOptions.DECIMAL_TO_DOUBLE, ExternalDataConstants.FALSE)); - timestampAsLong = Boolean.parseBoolean(configuration - .getOrDefault(ExternalDataConstants.IcebergOptions.TIMESTAMP_AS_LONG, ExternalDataConstants.TRUE)); - timeAsInt = Boolean.parseBoolean(configuration.getOrDefault(ExternalDataConstants.IcebergOptions.TIME_AS_INT, - ExternalDataConstants.TRUE)); - dateAsInt = Boolean.parseBoolean(configuration.getOrDefault(ExternalDataConstants.IcebergOptions.DATE_AS_INT, - ExternalDataConstants.TRUE)); + decimalToDouble = Boolean.parseBoolean(configuration.getOrDefault(DECIMAL_TO_DOUBLE, FALSE)); + dateAsInt = Boolean.parseBoolean(configuration.getOrDefault(DATE_AS_INT, FALSE)); + timeAsInt = Boolean.parseBoolean(configuration.getOrDefault(TIME_AS_INT, FALSE)); + timestampAsLong = Boolean.parseBoolean(configuration.getOrDefault(TIMESTAMP_AS_LONG, FALSE)); + String configuredTimeZoneId = configuration.get(ExternalDataConstants.IcebergOptions.TIMEZONE); if (configuredTimeZoneId != null && !configuredTimeZoneId.isEmpty()) { timeZoneId = TimeZone.getTimeZone(configuredTimeZoneId).toZoneId(); @@ -72,24 +58,6 @@ } } - public void serializeDate(int value, DataOutput output) { - try { - mutableDate.setValue(value); - dateSerDer.serialize(mutableDate, output); - } catch (HyracksDataException e) { - throw new IllegalStateException(e); - } - } - - public void serializeDateTime(long timestamp, DataOutput output) { - try { - mutableDateTime.setValue(timestamp); - datetimeSerDer.serialize(mutableDateTime, output); - } catch (HyracksDataException e) { - throw new IllegalStateException(e); - } - } - public boolean isDecimalToDoubleEnabled() { return decimalToDouble; } -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21303?usp=email To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings?usp=email Gerrit-MessageType: merged Gerrit-Project: asterixdb Gerrit-Branch: lumina Gerrit-Change-Id: I3073631ac331fa14749048f6fc143b753ac1db2d Gerrit-Change-Number: 21303 Gerrit-PatchSet: 4 Gerrit-Owner: Hussain Towaileb <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Hussain Towaileb <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]> Gerrit-Reviewer: Murtadha Hubail <[email protected]>
