This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 30e27eddf9 [core] Reject per level avro format with high precision
timestamps (#7892)
30e27eddf9 is described below
commit 30e27eddf9893a9018bbfb04e2c97e265368ea28
Author: Arnav Balyan <[email protected]>
AuthorDate: Tue May 19 07:55:16 2026 +0530
[core] Reject per level avro format with high precision timestamps (#7892)
---
.../org/apache/paimon/schema/SchemaValidation.java | 29 +++++++++++++++
.../apache/paimon/schema/SchemaValidationTest.java | 41 ++++++++++++++++++++++
2 files changed, 70 insertions(+)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
index 13ee86cdd0..4ffc3ec025 100644
--- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
+++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
@@ -186,6 +186,35 @@ public class SchemaValidation {
}
fileFormat.validateDataFields(new RowType(fieldsInNormalFile));
+ for (Map.Entry<Integer, String> entry :
options.fileFormatPerLevel().entrySet()) {
+ if (!"avro".equalsIgnoreCase(entry.getValue())) {
+ continue;
+ }
+ for (DataField field : fieldsInNormalFile) {
+ DataType type = field.type();
+ int precision = -1;
+ if (type instanceof TimestampType) {
+ precision = ((TimestampType) type).getPrecision();
+ } else if (type instanceof LocalZonedTimestampType) {
+ precision = ((LocalZonedTimestampType)
type).getPrecision();
+ }
+ if (precision > 6) {
+ throw new IllegalArgumentException(
+ String.format(
+ "'%s' entry '%d:avro' is incompatible with
column '%s' of type %s: "
+ + "Avro supports timestamp
precision up to 6, got %d. "
+ + "Either lower the column
precision, drop the per-level mapping for level %d, "
+ + "or use a different format
(parquet or orc) for that level.",
+ CoreOptions.FILE_FORMAT_PER_LEVEL.key(),
+ entry.getKey(),
+ field.name(),
+ type,
+ precision,
+ entry.getKey()));
+ }
+ }
+ }
+
// Check column names in schema
schema.fieldNames()
.forEach(
diff --git
a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java
b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java
index cec075724f..d518f79a20 100644
---
a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java
@@ -444,4 +444,45 @@ class SchemaValidationTest {
validateTableSchema(
new TableSchema(1, fields, 10, emptyList(),
singletonList("f1"), options, ""));
}
+
+ @Test
+ public void testFileFormatPerLevelRejectsIncompatibleSchema() {
+ List<DataField> fields =
+ Arrays.asList(
+ new DataField(0, "k", DataTypes.INT()),
+ new DataField(1, "v", DataTypes.TIMESTAMP(9)));
+ Map<String, String> options = new HashMap<>();
+ options.put(BUCKET.key(), String.valueOf(-1));
+ options.put(CoreOptions.FILE_FORMAT_PER_LEVEL.key(), "0:avro");
+
+ assertThatThrownBy(
+ () ->
+ validateTableSchema(
+ new TableSchema(
+ 1,
+ fields,
+ 10,
+ emptyList(),
+ singletonList("k"),
+ options,
+ "")))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("file.format.per.level")
+ .hasMessageContaining("0:avro")
+ .hasMessageContaining("TIMESTAMP");
+ }
+
+ @Test
+ public void testFileFormatPerLevelAcceptsCompatibleSchema() {
+ List<DataField> fields =
+ Arrays.asList(
+ new DataField(0, "k", DataTypes.INT()),
+ new DataField(1, "v", DataTypes.TIMESTAMP(9)));
+ Map<String, String> options = new HashMap<>();
+ options.put(BUCKET.key(), String.valueOf(-1));
+ options.put(CoreOptions.FILE_FORMAT_PER_LEVEL.key(), "0:parquet");
+
+ validateTableSchema(
+ new TableSchema(1, fields, 10, emptyList(),
singletonList("k"), options, ""));
+ }
}
