This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 6d109d4195 [core] remove nullable check for record-level.time-field
(#4837)
6d109d4195 is described below
commit 6d109d4195d78534eaa264c2b594ae8acbcf0af2
Author: xiangyu0xf <[email protected]>
AuthorDate: Mon Jan 6 15:53:39 2025 +0800
[core] remove nullable check for record-level.time-field (#4837)
---
.../main/java/org/apache/paimon/schema/SchemaValidation.java | 6 ------
.../org/apache/paimon/catalog/PrimaryKeyTableTestBase.java | 6 +++++-
.../java/org/apache/paimon/schema/SchemaValidationTest.java | 4 ++--
.../java/org/apache/paimon/table/RecordLevelExpireTest.java | 12 +++++++++++-
.../paimon/table/RecordLevelExpireWithMillisecondTest.java | 11 ++++++++++-
.../paimon/table/RecordLevelExpireWithTimestampBaseTest.java | 9 +++++++++
.../paimon/table/RecordLevelExpireWithTimestampLTZTest.java | 2 +-
.../paimon/table/RecordLevelExpireWithTimestampTest.java | 2 +-
8 files changed, 39 insertions(+), 13 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
index 4bddcdd72d..f0f9284ed7 100644
--- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
+++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
@@ -203,12 +203,6 @@ public class SchemaValidation {
"The record level time field type should be
one of INT, BIGINT, or TIMESTAMP, but field type is %s.",
dataType));
}
- if (dataType.isNullable()) {
- throw new IllegalArgumentException(
- String.format(
- "Time field %s for record-level expire should
be not null.",
- recordLevelTimeField));
- }
}
if (options.mergeEngine() == MergeEngine.FIRST_ROW) {
diff --git
a/paimon-core/src/test/java/org/apache/paimon/catalog/PrimaryKeyTableTestBase.java
b/paimon-core/src/test/java/org/apache/paimon/catalog/PrimaryKeyTableTestBase.java
index 54152c8965..e648699aed 100644
---
a/paimon-core/src/test/java/org/apache/paimon/catalog/PrimaryKeyTableTestBase.java
+++
b/paimon-core/src/test/java/org/apache/paimon/catalog/PrimaryKeyTableTestBase.java
@@ -123,7 +123,11 @@ public abstract class PrimaryKeyTableTestBase {
r -> {
GenericRow newR = new
GenericRow(projection.length);
for (int i = 0; i < projection.length; i++) {
- newR.setField(i, r.getInt(i));
+ if (r.isNullAt(i)) {
+ newR.setField(i, null);
+ } else {
+ newR.setField(i, r.getInt(i));
+ }
}
rows.add(newR);
});
diff --git
a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java
b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java
index 9dbfff200e..501b9f588d 100644
---
a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java
@@ -36,6 +36,7 @@ import java.util.Map;
import static org.apache.paimon.CoreOptions.BUCKET;
import static org.apache.paimon.CoreOptions.SCAN_SNAPSHOT_ID;
import static org.apache.paimon.schema.SchemaValidation.validateTableSchema;
+import static org.assertj.core.api.Assertions.assertThatCode;
import static org.assertj.core.api.Assertions.assertThatNoException;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -110,8 +111,7 @@ class SchemaValidationTest {
Map<String, String> options = new HashMap<>(2);
options.put(CoreOptions.RECORD_LEVEL_TIME_FIELD.key(), "f0");
options.put(CoreOptions.RECORD_LEVEL_EXPIRE_TIME.key(), "1 m");
- assertThatThrownBy(() -> validateTableSchemaExec(options))
- .hasMessageContaining("Time field f0 for record-level expire
should be not null");
+ assertThatCode(() ->
validateTableSchemaExec(options)).doesNotThrowAnyException();
options.put(CoreOptions.RECORD_LEVEL_TIME_FIELD.key(), "f10");
assertThatThrownBy(() -> validateTableSchemaExec(options))
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireTest.java
index 08c8ac5480..bd13b0ecf8 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireTest.java
@@ -38,6 +38,7 @@ import java.time.Duration;
import java.util.UUID;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
class RecordLevelExpireTest extends PrimaryKeyTableTestBase {
@@ -54,7 +55,7 @@ class RecordLevelExpireTest extends PrimaryKeyTableTestBase {
Schema.newBuilder()
.column("pt", DataTypes.INT())
.column("pk", DataTypes.INT())
- .column("col1", DataTypes.INT().notNull())
+ .column("col1", DataTypes.INT())
.partitionKeys("pt")
.primaryKey("pk", "pt")
.options(tableOptions().toMap())
@@ -99,5 +100,14 @@ class RecordLevelExpireTest extends PrimaryKeyTableTestBase
{
assertThat(query()).containsExactlyInAnyOrder(GenericRow.of(1, 4,
currentSecs + 60 * 60));
assertThat(query(new int[] {2}))
.containsExactlyInAnyOrder(GenericRow.of(currentSecs + 60 *
60));
+
+ writeCommit(GenericRow.of(1, 5, null));
+ assertThat(query())
+ .containsExactlyInAnyOrder(
+ GenericRow.of(1, 4, currentSecs + 60 * 60),
GenericRow.of(1, 5, null));
+
+ // null time field for record-level expire is not supported yet.
+ assertThatThrownBy(() -> compact(1))
+ .hasMessageContaining("Time field for record-level expire
should not be null.");
}
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireWithMillisecondTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireWithMillisecondTest.java
index 393683fc92..295058bfbd 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireWithMillisecondTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireWithMillisecondTest.java
@@ -38,6 +38,7 @@ import java.time.Duration;
import java.util.UUID;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
class RecordLevelExpireWithMillisecondTest extends PrimaryKeyTableTestBase {
@Override
@@ -53,7 +54,7 @@ class RecordLevelExpireWithMillisecondTest extends
PrimaryKeyTableTestBase {
Schema.newBuilder()
.column("pt", DataTypes.INT())
.column("pk", DataTypes.INT())
- .column("col1", DataTypes.BIGINT().notNull())
+ .column("col1", DataTypes.BIGINT())
.partitionKeys("pt")
.primaryKey("pk", "pt")
.options(tableOptions().toMap())
@@ -96,5 +97,13 @@ class RecordLevelExpireWithMillisecondTest extends
PrimaryKeyTableTestBase {
// compact, expired
compact(1);
assertThat(query(new int[] {0,
1})).containsExactlyInAnyOrder(GenericRow.of(1, 4));
+
+ writeCommit(GenericRow.of(1, 5, null));
+ assertThat(query(new int[] {0, 1}))
+ .containsExactlyInAnyOrder(GenericRow.of(1, 4),
GenericRow.of(1, 5));
+
+ // null time field for record-level expire is not supported yet.
+ assertThatThrownBy(() -> compact(1))
+ .hasMessageContaining("Time field for record-level expire
should not be null.");
}
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireWithTimestampBaseTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireWithTimestampBaseTest.java
index abcb8c1c76..f352693759 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireWithTimestampBaseTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireWithTimestampBaseTest.java
@@ -29,6 +29,7 @@ import org.junit.jupiter.api.Test;
import java.time.Duration;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
abstract class RecordLevelExpireWithTimestampBaseTest extends
PrimaryKeyTableTestBase {
@@ -61,5 +62,13 @@ abstract class RecordLevelExpireWithTimestampBaseTest
extends PrimaryKeyTableTes
// compact, expired
compact(1);
assertThat(query(new int[] {0,
1})).containsExactlyInAnyOrder(GenericRow.of(1, 3));
+
+ writeCommit(GenericRow.of(1, 5, null));
+ assertThat(query(new int[] {0, 1}))
+ .containsExactlyInAnyOrder(GenericRow.of(1, 3),
GenericRow.of(1, 5));
+
+ // null time field for record-level expire is not supported yet.
+ assertThatThrownBy(() -> compact(1))
+ .hasMessageContaining("Time field for record-level expire
should not be null.");
}
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireWithTimestampLTZTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireWithTimestampLTZTest.java
index c256171c6f..af834af276 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireWithTimestampLTZTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireWithTimestampLTZTest.java
@@ -46,7 +46,7 @@ class RecordLevelExpireWithTimestampLTZTest extends
RecordLevelExpireWithTimesta
Schema.newBuilder()
.column("pt", DataTypes.INT())
.column("pk", DataTypes.INT())
- .column("col1",
DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE().notNull())
+ .column("col1",
DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE())
.partitionKeys("pt")
.primaryKey("pk", "pt")
.options(tableOptions().toMap())
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireWithTimestampTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireWithTimestampTest.java
index 697053c0e7..3c4add8914 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireWithTimestampTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireWithTimestampTest.java
@@ -46,7 +46,7 @@ class RecordLevelExpireWithTimestampTest extends
RecordLevelExpireWithTimestampB
Schema.newBuilder()
.column("pt", DataTypes.INT())
.column("pk", DataTypes.INT())
- .column("col1", DataTypes.TIMESTAMP().notNull())
+ .column("col1", DataTypes.TIMESTAMP())
.partitionKeys("pt")
.primaryKey("pk", "pt")
.options(tableOptions().toMap())
