lasdf1234 commented on code in PR #14659:
URL: https://github.com/apache/gravitino/pull/14659#discussion_r4489725164
##########
clients/client-python/gravitino/api/rel/column.py:
##########
@@ -97,14 +101,28 @@ def auto_increment(self) -> bool:
@abstractmethod
def default_value(self) -> Expression:
- """Get the default value of this column
+ """Get the write-time default value of this column.
+
+ For Iceberg format-version 3, this corresponds to ``write-default``.
Returns:
Expression:
The default value of this column,
`Column.DEFAULT_VALUE_NOT_SET` if not specified.
"""
pass
+ def initial_default_value(self) -> Expression:
+ """Get the initial default value of this column.
+
+ For Iceberg format-version 3, this corresponds to ``initial-default``.
Catalogs that do
+ not distinguish an initial default may leave this unset.
+
+ Returns:
+ Expression:
+ The initial default value,
`Column.INITIAL_DEFAULT_VALUE_NOT_SET` if not specified.
+ """
+ return Column.INITIAL_DEFAULT_VALUE_NOT_SET
Review Comment:
Fixed in the latest revision.
`DTOConverters.from_dto(ColumnDTO)` now propagates `initial_default_value`
(including when `default_value` is set), `to_dto` writes it back, and
`GenericColumn.initial_default_value()` forwards to the underlying column.
##########
catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/converter/ToIcebergColumnDefault.java:
##########
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.catalog.lakehouse.iceberg.converter;
+
+import com.google.common.base.Preconditions;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.util.UUID;
+import org.apache.gravitino.rel.Column;
+import org.apache.gravitino.rel.expressions.Expression;
+import org.apache.gravitino.rel.expressions.literals.Literals;
+import org.apache.gravitino.rel.types.Decimal;
+import org.apache.iceberg.expressions.Literal;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.ByteBuffers;
+import org.apache.iceberg.util.DateTimeUtil;
+
+/**
+ * Converts Gravitino column default {@link Expression} values into Iceberg
{@link Literal} values
+ * for format-version 3 write-default / initial-default.
+ */
+public final class ToIcebergColumnDefault {
+
+ private ToIcebergColumnDefault() {}
+
+ /**
+ * Converts a Gravitino default expression to an Iceberg {@link Literal}, or
{@code null} when
+ * unset / NULL.
+ *
+ * @param expression the Gravitino expression
+ * @param icebergType the target Iceberg field type
+ * @return the Iceberg literal, or null when unset
+ */
+ public static Literal<?> fromGravitino(Expression expression, Type
icebergType) {
+ if (expression == null
+ || expression.equals(Column.DEFAULT_VALUE_NOT_SET)
+ || expression.equals(Column.INITIAL_DEFAULT_VALUE_NOT_SET)
+ || Literals.NULL.equals(expression)) {
+ return null;
+ }
+ Preconditions.checkArgument(
+ expression instanceof
org.apache.gravitino.rel.expressions.literals.Literal,
+ "Iceberg column defaults must be literals, got: %s",
+ expression);
+ Object value = ((org.apache.gravitino.rel.expressions.literals.Literal<?>)
expression).value();
Review Comment:
Agreed this was a real bug on the write path.
Per the scoped read-only change, `ToIcebergColumnDefault` has been removed
and Gravitino create/alter still does not set Iceberg column defaults
(`IcebergCatalogCapability.columnDefaultValue()` remains unsupported). This
ClassCastException path is therefore gone with that code. If/when we add write
support in a follow-up, we will parse REST literal strings by declared type
before converting to Iceberg.
##########
catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/converter/FromIcebergColumnDefault.java:
##########
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.catalog.lakehouse.iceberg.converter;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.UUID;
+import org.apache.gravitino.rel.Column;
+import org.apache.gravitino.rel.expressions.Expression;
+import org.apache.gravitino.rel.expressions.literals.Literals;
+import org.apache.gravitino.rel.types.Decimal;
+import org.apache.gravitino.rel.types.Types;
+import org.apache.iceberg.expressions.Literal;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types.DecimalType;
+import org.apache.iceberg.types.Types.FixedType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.TimestampNanoType;
+import org.apache.iceberg.types.Types.TimestampType;
+import org.apache.iceberg.util.ByteBuffers;
+import org.apache.iceberg.util.DateTimeUtil;
+
+/**
+ * Converts Iceberg format-version 3 column defaults into Gravitino {@link
Expression} values.
+ *
+ * <p>Iceberg {@code write-default} maps to {@link Column#defaultValue()} and
{@code
+ * initial-default} maps to {@link Column#initialDefaultValue()}.
+ */
+final class FromIcebergColumnDefault {
+
+ private FromIcebergColumnDefault() {}
+
+ /**
+ * Returns the Gravitino write-default expression for {@code field}.
+ *
+ * @param field the Iceberg nested field
+ * @return the mapped default expression
+ */
+ static Expression writeDefault(NestedField field) {
+ return toGravitino(field.writeDefaultLiteral(), field.type(),
Column.DEFAULT_VALUE_NOT_SET);
+ }
+
+ /**
+ * Returns the Gravitino initial-default expression for {@code field}.
+ *
+ * @param field the Iceberg nested field
+ * @return the mapped initial default expression
+ */
+ static Expression initialDefault(NestedField field) {
+ return toGravitino(
+ field.initialDefaultLiteral(), field.type(),
Column.INITIAL_DEFAULT_VALUE_NOT_SET);
+ }
+
+ private static Expression toGravitino(Literal<?> icebergDefault, Type type,
Expression unset) {
+ if (icebergDefault == null) {
+ return unset;
+ }
+ Object value = icebergDefault.value();
+ if (value == null) {
+ return Literals.NULL;
+ }
+
+ switch (type.typeId()) {
+ case BOOLEAN:
+ return Literals.booleanLiteral((Boolean) value);
+ case INTEGER:
+ return Literals.integerLiteral((Integer) value);
+ case LONG:
+ return Literals.longLiteral((Long) value);
+ case FLOAT:
+ return Literals.floatLiteral((Float) value);
+ case DOUBLE:
+ return Literals.doubleLiteral((Double) value);
+ case STRING:
+ return Literals.stringLiteral(value.toString());
+ case DATE:
+ return Literals.dateLiteral(DateTimeUtil.dateFromDays((Integer)
value));
+ case TIME:
+ return Literals.timeLiteral(DateTimeUtil.timeFromMicros((Long) value));
+ case TIMESTAMP:
+ {
+ TimestampType timestampType = (TimestampType) type;
+ if (timestampType.shouldAdjustToUTC()) {
+ return Literals.of(
+ DateTimeUtil.timestamptzFromMicros((Long)
value).toLocalDateTime(),
+
Types.TimestampType.withTimeZone(FromIcebergType.PRECISION_MICROSECOND));
+ }
+ return Literals.of(
+ DateTimeUtil.timestampFromMicros((Long) value),
+
Types.TimestampType.withoutTimeZone(FromIcebergType.PRECISION_MICROSECOND));
+ }
+ case TIMESTAMP_NANO:
+ {
+ TimestampNanoType timestampNanoType = (TimestampNanoType) type;
+ if (timestampNanoType.shouldAdjustToUTC()) {
+ return Literals.of(
+ DateTimeUtil.timestamptzFromNanos((Long)
value).toLocalDateTime(),
+
Types.TimestampType.withTimeZone(FromIcebergType.PRECISION_NANOSECOND));
+ }
+ return Literals.of(
+ DateTimeUtil.timestampFromNanos((Long) value),
+
Types.TimestampType.withoutTimeZone(FromIcebergType.PRECISION_NANOSECOND));
+ }
+ case DECIMAL:
+ {
+ DecimalType decimalType = (DecimalType) type;
+ return Literals.decimalLiteral(
+ Decimal.of((BigDecimal) value, decimalType.precision(),
decimalType.scale()));
+ }
+ case UUID:
+ return Literals.of(((UUID) value).toString(), Types.UUIDType.get());
+ case FIXED:
+ return Literals.of(toByteArray(value), Types.FixedType.of(((FixedType)
type).length()));
+ case BINARY:
+ return Literals.of(toByteArray(value), Types.BinaryType.get());
Review Comment:
`FromIcebergColumnDefault` was removed with the write-path cleanup;
Iceberg→Gravitino default mapping now lives in `ConvertUtil.fromNestedField` /
`toGravitinoLiteral`.
The underlying DTO issue you found is still valid for the **read** path:
`DTOConverters.toFunctionArg()` still uses `value.toString()` for literals, so
FIXED/BINARY defaults can still serialize as values like `[B@...` in API
responses. That is broader than this PR’s Iceberg load mapping and is not fixed
here. Happy to track a follow-up for a reversible binary/fixed literal encoding
in DTO/JSON.
##########
catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/converter/ToIcebergColumnDefault.java:
##########
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.catalog.lakehouse.iceberg.converter;
+
+import com.google.common.base.Preconditions;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.util.UUID;
+import org.apache.gravitino.rel.Column;
+import org.apache.gravitino.rel.expressions.Expression;
+import org.apache.gravitino.rel.expressions.literals.Literals;
+import org.apache.gravitino.rel.types.Decimal;
+import org.apache.iceberg.expressions.Literal;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.ByteBuffers;
+import org.apache.iceberg.util.DateTimeUtil;
+
+/**
+ * Converts Gravitino column default {@link Expression} values into Iceberg
{@link Literal} values
+ * for format-version 3 write-default / initial-default.
+ */
+public final class ToIcebergColumnDefault {
+
+ private ToIcebergColumnDefault() {}
+
+ /**
+ * Converts a Gravitino default expression to an Iceberg {@link Literal}, or
{@code null} when
+ * unset / NULL.
+ *
+ * @param expression the Gravitino expression
+ * @param icebergType the target Iceberg field type
+ * @return the Iceberg literal, or null when unset
+ */
+ public static Literal<?> fromGravitino(Expression expression, Type
icebergType) {
+ if (expression == null
+ || expression.equals(Column.DEFAULT_VALUE_NOT_SET)
+ || expression.equals(Column.INITIAL_DEFAULT_VALUE_NOT_SET)
+ || Literals.NULL.equals(expression)) {
+ return null;
+ }
+ Preconditions.checkArgument(
+ expression instanceof
org.apache.gravitino.rel.expressions.literals.Literal,
+ "Iceberg column defaults must be literals, got: %s",
+ expression);
+ Object value = ((org.apache.gravitino.rel.expressions.literals.Literal<?>)
expression).value();
+ Literal<?> icebergLiteral;
+ switch (icebergType.typeId()) {
+ case BOOLEAN:
+ icebergLiteral = Literal.of((Boolean) value);
+ break;
+ case INTEGER:
+ icebergLiteral = Literal.of(((Number) value).intValue());
+ break;
+ case LONG:
+ icebergLiteral = Literal.of(((Number) value).longValue());
+ break;
+ case FLOAT:
+ icebergLiteral = Literal.of(((Number) value).floatValue());
+ break;
+ case DOUBLE:
+ icebergLiteral = Literal.of(((Number) value).doubleValue());
+ break;
+ case STRING:
+ icebergLiteral = Literal.of(String.valueOf(value));
+ break;
+ case DATE:
+ icebergLiteral = Literal.of(DateTimeUtil.daysFromDate((LocalDate)
value));
+ break;
+ case TIME:
+ icebergLiteral = Literal.of(DateTimeUtil.microsFromTime((LocalTime)
value));
+ break;
+ case TIMESTAMP:
+ {
+ Types.TimestampType timestampType = (Types.TimestampType)
icebergType;
+ long micros = DateTimeUtil.microsFromTimestamp((LocalDateTime)
value);
+ icebergLiteral =
+ timestampType.shouldAdjustToUTC()
+ ? Literal.of(micros).to(timestampType)
+ : Literal.of(micros);
+ break;
+ }
+ case TIMESTAMP_NANO:
+ {
+ Types.TimestampNanoType timestampNanoType =
(Types.TimestampNanoType) icebergType;
+ icebergLiteral =
+ Literal.of(DateTimeUtil.nanosFromTimestamp((LocalDateTime)
value))
+ .to(timestampNanoType);
Review Comment:
Agreed on the double-scaling bug for timestamp-nano on the write path.
`ToIcebergColumnDefault` has been removed with the read-only scope, so this
conversion path is no longer in the PR. When write support is proposed later,
we will construct the nano literal without the extra micro→nano scale and add a
round-trip test.
##########
catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/converter/ToIcebergColumnDefault.java:
##########
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.catalog.lakehouse.iceberg.converter;
+
+import com.google.common.base.Preconditions;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.util.UUID;
+import org.apache.gravitino.rel.Column;
+import org.apache.gravitino.rel.expressions.Expression;
+import org.apache.gravitino.rel.expressions.literals.Literals;
+import org.apache.gravitino.rel.types.Decimal;
+import org.apache.iceberg.expressions.Literal;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.ByteBuffers;
+import org.apache.iceberg.util.DateTimeUtil;
+
+/**
+ * Converts Gravitino column default {@link Expression} values into Iceberg
{@link Literal} values
+ * for format-version 3 write-default / initial-default.
+ */
+public final class ToIcebergColumnDefault {
+
+ private ToIcebergColumnDefault() {}
+
+ /**
+ * Converts a Gravitino default expression to an Iceberg {@link Literal}, or
{@code null} when
+ * unset / NULL.
+ *
+ * @param expression the Gravitino expression
+ * @param icebergType the target Iceberg field type
+ * @return the Iceberg literal, or null when unset
+ */
+ public static Literal<?> fromGravitino(Expression expression, Type
icebergType) {
+ if (expression == null
+ || expression.equals(Column.DEFAULT_VALUE_NOT_SET)
+ || expression.equals(Column.INITIAL_DEFAULT_VALUE_NOT_SET)
+ || Literals.NULL.equals(expression)) {
+ return null;
+ }
+ Preconditions.checkArgument(
+ expression instanceof
org.apache.gravitino.rel.expressions.literals.Literal,
+ "Iceberg column defaults must be literals, got: %s",
+ expression);
+ Object value = ((org.apache.gravitino.rel.expressions.literals.Literal<?>)
expression).value();
+ Literal<?> icebergLiteral;
+ switch (icebergType.typeId()) {
+ case BOOLEAN:
+ icebergLiteral = Literal.of((Boolean) value);
+ break;
+ case INTEGER:
+ icebergLiteral = Literal.of(((Number) value).intValue());
Review Comment:
Agreed — silent narrowing/wrapping on overflow was incorrect for write
conversion.
`ToIcebergColumnDefault` has been removed with the read-only scope, so this
path is gone from the PR. If we add write support later, we will reject
out-of-range values instead of wrapping and add an overflow regression test.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]