mchades commented on code in PR #14659:
URL: https://github.com/apache/gravitino/pull/14659#discussion_r4488273291
##########
clients/client-python/gravitino/api/rel/column.py:
##########
@@ -97,14 +101,28 @@ def auto_increment(self) -> bool:
@abstractmethod
def default_value(self) -> Expression:
- """Get the default value of this column
+ """Get the write-time default value of this column.
+
+ For Iceberg format-version 3, this corresponds to ``write-default``.
Returns:
Expression:
The default value of this column,
`Column.DEFAULT_VALUE_NOT_SET` if not specified.
"""
pass
+ def initial_default_value(self) -> Expression:
+ """Get the initial default value of this column.
+
+ For Iceberg format-version 3, this corresponds to ``initial-default``.
Catalogs that do
+ not distinguish an initial default may leave this unset.
+
+ Returns:
+ Expression:
+ The initial default value,
`Column.INITIAL_DEFAULT_VALUE_NOT_SET` if not specified.
+ """
+ return Column.INITIAL_DEFAULT_VALUE_NOT_SET
Review Comment:
The Python client still loses this value:
`DTOConverters.from_dto(ColumnDTO)` drops it when `defaultValue` is present,
and `GenericColumn` inherits this fallback even when its underlying DTO retains
it. I reproduced a response with `initialDefaultValue="US"` returning NOT_SET
from `table.columns()[0].initial_default_value()`. Please propagate the field
through `from_dto`/`to_dto` and forward the getter in `GenericColumn`.
##########
catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/converter/ToIcebergColumnDefault.java:
##########
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.catalog.lakehouse.iceberg.converter;
+
+import com.google.common.base.Preconditions;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.util.UUID;
+import org.apache.gravitino.rel.Column;
+import org.apache.gravitino.rel.expressions.Expression;
+import org.apache.gravitino.rel.expressions.literals.Literals;
+import org.apache.gravitino.rel.types.Decimal;
+import org.apache.iceberg.expressions.Literal;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.ByteBuffers;
+import org.apache.iceberg.util.DateTimeUtil;
+
+/**
+ * Converts Gravitino column default {@link Expression} values into Iceberg
{@link Literal} values
+ * for format-version 3 write-default / initial-default.
+ */
+public final class ToIcebergColumnDefault {
+
+ private ToIcebergColumnDefault() {}
+
+ /**
+ * Converts a Gravitino default expression to an Iceberg {@link Literal}, or
{@code null} when
+ * unset / NULL.
+ *
+ * @param expression the Gravitino expression
+ * @param icebergType the target Iceberg field type
+ * @return the Iceberg literal, or null when unset
+ */
+ public static Literal<?> fromGravitino(Expression expression, Type
icebergType) {
+ if (expression == null
+ || expression.equals(Column.DEFAULT_VALUE_NOT_SET)
+ || expression.equals(Column.INITIAL_DEFAULT_VALUE_NOT_SET)
+ || Literals.NULL.equals(expression)) {
+ return null;
+ }
+ Preconditions.checkArgument(
+ expression instanceof
org.apache.gravitino.rel.expressions.literals.Literal,
+ "Iceberg column defaults must be literals, got: %s",
+ expression);
+ Object value = ((org.apache.gravitino.rel.expressions.literals.Literal<?>)
expression).value();
+ Literal<?> icebergLiteral;
+ switch (icebergType.typeId()) {
+ case BOOLEAN:
+ icebergLiteral = Literal.of((Boolean) value);
+ break;
+ case INTEGER:
+ icebergLiteral = Literal.of(((Number) value).intValue());
Review Comment:
This narrows the value before Iceberg can validate its range. I reproduced
`Literals.longLiteral(2147483648L)` for an integer column becoming
`-2147483648`, which is then accepted as the default. Please reject
out-of-range values instead of silently wrapping them, and add an overflow
regression test.
##########
catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/converter/ToIcebergColumnDefault.java:
##########
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.catalog.lakehouse.iceberg.converter;
+
+import com.google.common.base.Preconditions;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.util.UUID;
+import org.apache.gravitino.rel.Column;
+import org.apache.gravitino.rel.expressions.Expression;
+import org.apache.gravitino.rel.expressions.literals.Literals;
+import org.apache.gravitino.rel.types.Decimal;
+import org.apache.iceberg.expressions.Literal;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.ByteBuffers;
+import org.apache.iceberg.util.DateTimeUtil;
+
+/**
+ * Converts Gravitino column default {@link Expression} values into Iceberg
{@link Literal} values
+ * for format-version 3 write-default / initial-default.
+ */
+public final class ToIcebergColumnDefault {
+
+ private ToIcebergColumnDefault() {}
+
+ /**
+ * Converts a Gravitino default expression to an Iceberg {@link Literal}, or
{@code null} when
+ * unset / NULL.
+ *
+ * @param expression the Gravitino expression
+ * @param icebergType the target Iceberg field type
+ * @return the Iceberg literal, or null when unset
+ */
+ public static Literal<?> fromGravitino(Expression expression, Type
icebergType) {
+ if (expression == null
+ || expression.equals(Column.DEFAULT_VALUE_NOT_SET)
+ || expression.equals(Column.INITIAL_DEFAULT_VALUE_NOT_SET)
+ || Literals.NULL.equals(expression)) {
+ return null;
+ }
+ Preconditions.checkArgument(
+ expression instanceof
org.apache.gravitino.rel.expressions.literals.Literal,
+ "Iceberg column defaults must be literals, got: %s",
+ expression);
+ Object value = ((org.apache.gravitino.rel.expressions.literals.Literal<?>)
expression).value();
Review Comment:
REST literals reach this method as strings:
`DTOConverters.fromFunctionArg()` preserves `LiteralDTO.value()`. An integer
default `"8"` therefore throws `ClassCastException` at the `Number` cast;
boolean, decimal and date defaults fail similarly. Please parse the literal
according to its declared type before converting to the Iceberg type, and cover
the DTO-to-column-to-schema path in tests.
##########
catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/converter/FromIcebergColumnDefault.java:
##########
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.catalog.lakehouse.iceberg.converter;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.UUID;
+import org.apache.gravitino.rel.Column;
+import org.apache.gravitino.rel.expressions.Expression;
+import org.apache.gravitino.rel.expressions.literals.Literals;
+import org.apache.gravitino.rel.types.Decimal;
+import org.apache.gravitino.rel.types.Types;
+import org.apache.iceberg.expressions.Literal;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types.DecimalType;
+import org.apache.iceberg.types.Types.FixedType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.TimestampNanoType;
+import org.apache.iceberg.types.Types.TimestampType;
+import org.apache.iceberg.util.ByteBuffers;
+import org.apache.iceberg.util.DateTimeUtil;
+
+/**
+ * Converts Iceberg format-version 3 column defaults into Gravitino {@link
Expression} values.
+ *
+ * <p>Iceberg {@code write-default} maps to {@link Column#defaultValue()} and
{@code
+ * initial-default} maps to {@link Column#initialDefaultValue()}.
+ */
+final class FromIcebergColumnDefault {
+
+ private FromIcebergColumnDefault() {}
+
+ /**
+ * Returns the Gravitino write-default expression for {@code field}.
+ *
+ * @param field the Iceberg nested field
+ * @return the mapped default expression
+ */
+ static Expression writeDefault(NestedField field) {
+ return toGravitino(field.writeDefaultLiteral(), field.type(),
Column.DEFAULT_VALUE_NOT_SET);
+ }
+
+ /**
+ * Returns the Gravitino initial-default expression for {@code field}.
+ *
+ * @param field the Iceberg nested field
+ * @return the mapped initial default expression
+ */
+ static Expression initialDefault(NestedField field) {
+ return toGravitino(
+ field.initialDefaultLiteral(), field.type(),
Column.INITIAL_DEFAULT_VALUE_NOT_SET);
+ }
+
+ private static Expression toGravitino(Literal<?> icebergDefault, Type type,
Expression unset) {
+ if (icebergDefault == null) {
+ return unset;
+ }
+ Object value = icebergDefault.value();
+ if (value == null) {
+ return Literals.NULL;
+ }
+
+ switch (type.typeId()) {
+ case BOOLEAN:
+ return Literals.booleanLiteral((Boolean) value);
+ case INTEGER:
+ return Literals.integerLiteral((Integer) value);
+ case LONG:
+ return Literals.longLiteral((Long) value);
+ case FLOAT:
+ return Literals.floatLiteral((Float) value);
+ case DOUBLE:
+ return Literals.doubleLiteral((Double) value);
+ case STRING:
+ return Literals.stringLiteral(value.toString());
+ case DATE:
+ return Literals.dateLiteral(DateTimeUtil.dateFromDays((Integer)
value));
+ case TIME:
+ return Literals.timeLiteral(DateTimeUtil.timeFromMicros((Long) value));
+ case TIMESTAMP:
+ {
+ TimestampType timestampType = (TimestampType) type;
+ if (timestampType.shouldAdjustToUTC()) {
+ return Literals.of(
+ DateTimeUtil.timestamptzFromMicros((Long)
value).toLocalDateTime(),
+
Types.TimestampType.withTimeZone(FromIcebergType.PRECISION_MICROSECOND));
+ }
+ return Literals.of(
+ DateTimeUtil.timestampFromMicros((Long) value),
+
Types.TimestampType.withoutTimeZone(FromIcebergType.PRECISION_MICROSECOND));
+ }
+ case TIMESTAMP_NANO:
+ {
+ TimestampNanoType timestampNanoType = (TimestampNanoType) type;
+ if (timestampNanoType.shouldAdjustToUTC()) {
+ return Literals.of(
+ DateTimeUtil.timestamptzFromNanos((Long)
value).toLocalDateTime(),
+
Types.TimestampType.withTimeZone(FromIcebergType.PRECISION_NANOSECOND));
+ }
+ return Literals.of(
+ DateTimeUtil.timestampFromNanos((Long) value),
+
Types.TimestampType.withoutTimeZone(FromIcebergType.PRECISION_NANOSECOND));
+ }
+ case DECIMAL:
+ {
+ DecimalType decimalType = (DecimalType) type;
+ return Literals.decimalLiteral(
+ Decimal.of((BigDecimal) value, decimalType.precision(),
decimalType.scale()));
+ }
+ case UUID:
+ return Literals.of(((UUID) value).toString(), Types.UUIDType.get());
+ case FIXED:
+ return Literals.of(toByteArray(value), Types.FixedType.of(((FixedType)
type).length()));
+ case BINARY:
+ return Literals.of(toByteArray(value), Types.BinaryType.get());
Review Comment:
The `byte[]` returned here is not preserved by
`DTOConverters.toFunctionArg()`, which serializes `value.toString()`. I
reproduced a binary default containing bytes `[1, 2, 3]` returning JSON values
such as `"[B@42f93a98"` for the defaults, losing the original bytes. Please
provide a reversible binary/fixed representation through DTO/JSON and add a
round-trip test.
##########
catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/converter/ToIcebergColumnDefault.java:
##########
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.catalog.lakehouse.iceberg.converter;
+
+import com.google.common.base.Preconditions;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.util.UUID;
+import org.apache.gravitino.rel.Column;
+import org.apache.gravitino.rel.expressions.Expression;
+import org.apache.gravitino.rel.expressions.literals.Literals;
+import org.apache.gravitino.rel.types.Decimal;
+import org.apache.iceberg.expressions.Literal;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.ByteBuffers;
+import org.apache.iceberg.util.DateTimeUtil;
+
+/**
+ * Converts Gravitino column default {@link Expression} values into Iceberg
{@link Literal} values
+ * for format-version 3 write-default / initial-default.
+ */
+public final class ToIcebergColumnDefault {
+
+ private ToIcebergColumnDefault() {}
+
+ /**
+ * Converts a Gravitino default expression to an Iceberg {@link Literal}, or
{@code null} when
+ * unset / NULL.
+ *
+ * @param expression the Gravitino expression
+ * @param icebergType the target Iceberg field type
+ * @return the Iceberg literal, or null when unset
+ */
+ public static Literal<?> fromGravitino(Expression expression, Type
icebergType) {
+ if (expression == null
+ || expression.equals(Column.DEFAULT_VALUE_NOT_SET)
+ || expression.equals(Column.INITIAL_DEFAULT_VALUE_NOT_SET)
+ || Literals.NULL.equals(expression)) {
+ return null;
+ }
+ Preconditions.checkArgument(
+ expression instanceof
org.apache.gravitino.rel.expressions.literals.Literal,
+ "Iceberg column defaults must be literals, got: %s",
+ expression);
+ Object value = ((org.apache.gravitino.rel.expressions.literals.Literal<?>)
expression).value();
+ Literal<?> icebergLiteral;
+ switch (icebergType.typeId()) {
+ case BOOLEAN:
+ icebergLiteral = Literal.of((Boolean) value);
+ break;
+ case INTEGER:
+ icebergLiteral = Literal.of(((Number) value).intValue());
+ break;
+ case LONG:
+ icebergLiteral = Literal.of(((Number) value).longValue());
+ break;
+ case FLOAT:
+ icebergLiteral = Literal.of(((Number) value).floatValue());
+ break;
+ case DOUBLE:
+ icebergLiteral = Literal.of(((Number) value).doubleValue());
+ break;
+ case STRING:
+ icebergLiteral = Literal.of(String.valueOf(value));
+ break;
+ case DATE:
+ icebergLiteral = Literal.of(DateTimeUtil.daysFromDate((LocalDate)
value));
+ break;
+ case TIME:
+ icebergLiteral = Literal.of(DateTimeUtil.microsFromTime((LocalTime)
value));
+ break;
+ case TIMESTAMP:
+ {
+ Types.TimestampType timestampType = (Types.TimestampType)
icebergType;
+ long micros = DateTimeUtil.microsFromTimestamp((LocalDateTime)
value);
+ icebergLiteral =
+ timestampType.shouldAdjustToUTC()
+ ? Literal.of(micros).to(timestampType)
+ : Literal.of(micros);
+ break;
+ }
+ case TIMESTAMP_NANO:
+ {
+ Types.TimestampNanoType timestampNanoType =
(Types.TimestampNanoType) icebergType;
+ icebergLiteral =
+ Literal.of(DateTimeUtil.nanosFromTimestamp((LocalDateTime)
value))
+ .to(timestampNanoType);
Review Comment:
`Literal.of(long).to(timestampNanoType)` interprets the long as microseconds
and converts it to nanoseconds, but `nanosFromTimestamp()` has already produced
nanoseconds. I reproduced `1123456789` becoming `1123456789000`;
`2026-11-06T00:00:00.123456789` throws `ArithmeticException: long overflow`.
Please construct the nano literal without the second scaling step and add a
round-trip test.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]