This is an automated email from the ASF dual-hosted git repository. XiaoHongbo-Hope pushed a commit to branch release-2.1 in repository https://gitbox.apache.org/repos/asf/paimon.git
commit e69c4c86895a41bb7174546bfaf8d9aad47f83a6 Author: Xiangyi Zhu <[email protected]> AuthorDate: Thu Sep 17 17:08:43 2026 +0800 [common] Reject out-of-range integral casts when extracting a variant scalar (#9809) --- .../paimon/data/variant/BaseVariantReader.java | 7 +- .../org/apache/paimon/data/variant/VariantGet.java | 88 ++++++++++++++++++++-- .../paimon/data/variant/GenericVariantTest.java | 37 +++++++++ .../parquet/reader/VariantShreddingReadTest.java | 50 ++++++++++++ .../apache/paimon/spark/sql/VariantTestBase.scala | 14 ++++ 5 files changed, 183 insertions(+), 13 deletions(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/data/variant/BaseVariantReader.java b/paimon-common/src/main/java/org/apache/paimon/data/variant/BaseVariantReader.java index 0d0177c607..e5c5011d20 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/variant/BaseVariantReader.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/variant/BaseVariantReader.java @@ -543,11 +543,8 @@ public class BaseVariantReader { if (noNeedCast) { return i; } - try { - return resolve.cast(i); - } catch (Exception e) { - return invalidCast(row, topLevelMetadata); - } + Object result = VariantGet.castScalar(i, scalaType, targetType, resolve); + return result == null ? invalidCast(row, topLevelMetadata) : result; } } } diff --git a/paimon-common/src/main/java/org/apache/paimon/data/variant/VariantGet.java b/paimon-common/src/main/java/org/apache/paimon/data/variant/VariantGet.java index 7e355939eb..5a22fc1752 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/variant/VariantGet.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/variant/VariantGet.java @@ -34,7 +34,10 @@ import org.apache.paimon.types.MapType; import org.apache.paimon.types.RowType; import org.apache.paimon.types.VariantType; +import javax.annotation.Nullable; + import java.math.BigDecimal; +import java.math.RoundingMode; import java.util.HashMap; /** Utils for variant get. */ @@ -172,15 +175,84 @@ public class VariantGet { CastExecutor<Object, Object> resolve = (CastExecutor<Object, Object>) CastExecutors.resolve(inputType, dataType); - if (resolve != null) { - try { - return resolve.cast(input); - } catch (Exception e) { - return invalidCast(v, dataType, castArgs); - } - } + Object result = castScalar(input, inputType, dataType, resolve); + return result == null ? invalidCast(v, dataType, castArgs) : result; + } + } - return invalidCast(v, dataType, castArgs); + /** + * Casts a non-null scalar read from a variant to {@code targetType}, returning null when the + * cast is invalid. The generic cast rules wrap a numeric value that does not fit the target, so + * an out-of-range value is rejected here first, matching Spark's TRY cast semantics. + */ + @Nullable + static Object castScalar( + Object input, + DataType inputType, + DataType targetType, + @Nullable CastExecutor<Object, Object> executor) { + if (executor == null || !fitsIntegralTarget(input, inputType, targetType)) { + return null; + } + try { + return executor.cast(input); + } catch (Exception e) { + return null; + } + } + + /** Whether a numeric {@code input} lies within the range of an integral {@code targetType}. */ + private static boolean fitsIntegralTarget( + Object input, DataType inputType, DataType targetType) { + long min; + long max; + switch (targetType.getTypeRoot()) { + case TINYINT: + min = Byte.MIN_VALUE; + max = Byte.MAX_VALUE; + break; + case SMALLINT: + min = Short.MIN_VALUE; + max = Short.MAX_VALUE; + break; + case INTEGER: + min = Integer.MIN_VALUE; + max = Integer.MAX_VALUE; + break; + case BIGINT: + min = Long.MIN_VALUE; + max = Long.MAX_VALUE; + break; + default: + return true; + } + + switch (inputType.getTypeRoot()) { + case TINYINT: + case SMALLINT: + case INTEGER: + case BIGINT: + long value = ((Number) input).longValue(); + return value >= min && value <= max; + case FLOAT: + case DOUBLE: + // The fractional part is truncated by the cast, so any finite value strictly + // between min - 1 and max + 1 fits. Both bounds are exact doubles; for BIGINT + // max + 1 is 2^63 and min - 1 rounds to -2^63, which is itself in range. + double d = ((Number) input).doubleValue(); + if (Double.isNaN(d) || Double.isInfinite(d)) { + return false; + } + return max == Long.MAX_VALUE + ? d >= -0x1p63 && d < 0x1p63 + : d > min - 1.0 && d < max + 1.0; + case DECIMAL: + BigDecimal truncated = + ((Decimal) input).toBigDecimal().setScale(0, RoundingMode.DOWN); + return truncated.compareTo(BigDecimal.valueOf(min)) >= 0 + && truncated.compareTo(BigDecimal.valueOf(max)) <= 0; + default: + return true; } } diff --git a/paimon-common/src/test/java/org/apache/paimon/data/variant/GenericVariantTest.java b/paimon-common/src/test/java/org/apache/paimon/data/variant/GenericVariantTest.java index e5e507c455..46b5b8c992 100644 --- a/paimon-common/src/test/java/org/apache/paimon/data/variant/GenericVariantTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/data/variant/GenericVariantTest.java @@ -41,6 +41,7 @@ import static org.apache.paimon.data.variant.PaimonShreddingUtils.buildVariantSc import static org.apache.paimon.data.variant.PaimonShreddingUtils.castShredded; import static org.apache.paimon.data.variant.PaimonShreddingUtils.variantShreddingSchema; import static org.apache.paimon.types.DataTypesTest.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; /** Test of {@link GenericVariant}. */ public class GenericVariantTest { @@ -308,6 +309,42 @@ public class GenericVariantTest { .isEqualTo(Decimal.fromBigDecimal(new BigDecimal("100.0"), 5, 1)); } + @Test + public void testVariantGetIntegralOverflow() { + // The generic cast rules wrap or saturate a value that does not fit the target, which + // would turn an invalid cast into a wrong number; Spark's TRY cast rejects it instead. + Variant variant = + GenericVariant.fromJson( + "{\"big\": 99999999999, \"huge\": 1e30, \"negHuge\": -1e30," + + " \"wide\": 12345678901234567890.5, \"nan\": 1.5," + + " \"fits\": 2147483647, \"neg\": -2147483648.5}"); + VariantCastArgs tryCast = new VariantCastArgs(false, ZoneOffset.UTC); + VariantCastArgs strict = new VariantCastArgs(true, ZoneOffset.UTC); + + assertThat(variant.variantGet("$.big", DataTypes.INT(), tryCast)).isNull(); + assertThat(variant.variantGet("$.big", DataTypes.SMALLINT(), tryCast)).isNull(); + assertThat(variant.variantGet("$.big", DataTypes.TINYINT(), tryCast)).isNull(); + assertThat(variant.variantGet("$.huge", DataTypes.BIGINT(), tryCast)).isNull(); + assertThat(variant.variantGet("$.huge", DataTypes.INT(), tryCast)).isNull(); + assertThat(variant.variantGet("$.negHuge", DataTypes.INT(), tryCast)).isNull(); + assertThat(variant.variantGet("$.wide", DataTypes.BIGINT(), tryCast)).isNull(); + assertThatThrownBy(() -> variant.variantGet("$.big", DataTypes.INT(), strict)) + .hasMessageContaining("Invalid cast 99999999999 to INT"); + assertThatThrownBy(() -> variant.variantGet("$.huge", DataTypes.BIGINT(), strict)) + .hasMessageContaining("Invalid cast 1.0E30 to BIGINT"); + + // in-range values still truncate the fractional part and widen as before + assertThat(variant.variantGet("$.big", DataTypes.BIGINT(), tryCast)) + .isEqualTo(99999999999L); + assertThat(variant.variantGet("$.nan", DataTypes.INT(), tryCast)).isEqualTo(1); + assertThat(variant.variantGet("$.fits", DataTypes.INT(), tryCast)) + .isEqualTo(Integer.MAX_VALUE); + assertThat(variant.variantGet("$.neg", DataTypes.INT(), tryCast)) + .isEqualTo(Integer.MIN_VALUE); + assertThat(variant.variantGet("$.wide", DataTypes.DOUBLE(), tryCast)) + .isEqualTo(1.2345678901234567E19); + } + @Test public void testObjectFieldOrderingCompatibility() { String bmpKey = "\uE000"; diff --git a/paimon-format/src/test/java/org/apache/paimon/format/parquet/reader/VariantShreddingReadTest.java b/paimon-format/src/test/java/org/apache/paimon/format/parquet/reader/VariantShreddingReadTest.java index 34c4bdd382..5b93ff3c73 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/parquet/reader/VariantShreddingReadTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/parquet/reader/VariantShreddingReadTest.java @@ -54,6 +54,7 @@ import java.util.List; import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; /** Tests for variant shredding read. */ public class VariantShreddingReadTest { @@ -544,6 +545,55 @@ public class VariantShreddingReadTest { 0.0)); } + @ParameterizedTest + @ValueSource( + strings = { + "null", + "{\"type\":\"ROW\",\"fields\":[{\"name\":\"v\",\"type\":{\"type\":\"ROW\",\"fields\":[{\"name\":\"n\",\"type\":\"BIGINT\"},{\"name\":\"d\",\"type\":\"DOUBLE\"}]}}]}" + }) + public void testReadIntegralOverflowAsInvalidCast(String shreddingSchema) throws Exception { + // A shredded typed_value is narrowed by the scalar reader and an unshredded value by + // VariantGet; both must reject an out-of-range number instead of wrapping it. + Options options = new Options(); + if (!shreddingSchema.equals("null")) { + options.set("parquet.variant.shreddingSchema", shreddingSchema); + } + ParquetFileFormat format = + new ParquetFileFormat(new FileFormatFactory.FormatContext(options, 1024, 1024)); + + RowType writeType = DataTypes.ROW(DataTypes.FIELD(0, "v", DataTypes.VARIANT())); + writeRows( + format.createWriterFactory(writeType), + GenericRow.of(GenericVariant.fromJson("{\"n\":99999999999,\"d\":1e30}")), + GenericRow.of(GenericVariant.fromJson("{\"n\":7,\"d\":1.5}"))); + + RowType tryReadType = + DataTypes.ROW( + DataTypes.FIELD( + 0, + "v", + VariantMetadataUtils.VariantRowTypeBuilder.builder() + .field(DataTypes.INT(), "$.n", false, "UTC") + .field(DataTypes.BIGINT(), "$.d", false, "UTC") + .build())); + List<InternalRow> result = readRows(format, tryReadType); + assertThat(result.get(0).getRow(0, 2).isNullAt(0)).isTrue(); + assertThat(result.get(0).getRow(0, 2).isNullAt(1)).isTrue(); + assertThat(result.get(1).getRow(0, 2).getInt(0)).isEqualTo(7); + assertThat(result.get(1).getRow(0, 2).getLong(1)).isEqualTo(1L); + + RowType strictReadType = + DataTypes.ROW( + DataTypes.FIELD( + 0, + "v", + VariantMetadataUtils.VariantRowTypeBuilder.builder() + .field(DataTypes.INT(), "$.n", true, "UTC") + .build())); + assertThatThrownBy(() -> readRows(format, strictReadType)) + .hasMessageContaining("Invalid cast 99999999999 to INT"); + } + protected List<InternalRow> readRows(ParquetFileFormat format, RowType rowType) throws IOException { List<InternalRow> result = new ArrayList<>(); diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala index 0b50b12870..4501713510 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala @@ -1099,6 +1099,20 @@ abstract class VariantTestBase extends PaimonSparkTestBase { ) } + test("Paimon Variant pushdown: out-of-range extraction is NULL rather than a wrapped value") { + sql("CREATE TABLE T (id INT, v VARIANT)") + sql("""INSERT INTO T VALUES + | (1, parse_json('{"n":99999999999,"d":1e30}')), + | (2, parse_json('{"n":7,"d":1.5}')) + |""".stripMargin) + + // With pushdown the cast runs in the Paimon reader, which must agree with Spark's TRY cast. + checkAnswer( + sql( + "SELECT id, try_variant_get(v, '$.n', 'int'), try_variant_get(v, '$.d', 'bigint') FROM T ORDER BY id"), + Seq(Row(1, null, null), Row(2, 7, 1L))) + } + test("Paimon Variant pushdown: nested variant column inside a struct") { assume(gteqSpark4_1) sql("CREATE TABLE T (id INT, nested STRUCT<v: VARIANT, x: INT>)")
