This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 4bf2d9b01d [flink] Fix that 'values-time' partition expire might throw
NPE (#4646)
4bf2d9b01d is described below
commit 4bf2d9b01d0176bdf52e68dd996cbe79f4304d14
Author: yuzelin <[email protected]>
AuthorDate: Thu Dec 5 23:57:05 2024 +0800
[flink] Fix that 'values-time' partition expire might throw NPE (#4646)
---
.../PartitionValuesTimeExpireStrategy.java | 21 ++++++++++++++++-----
.../procedure/ExpirePartitionsProcedureITCase.java | 13 +++++++++++++
2 files changed, 29 insertions(+), 5 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/partition/PartitionValuesTimeExpireStrategy.java
b/paimon-core/src/main/java/org/apache/paimon/partition/PartitionValuesTimeExpireStrategy.java
index 80ae633fd2..51c53282c4 100644
---
a/paimon-core/src/main/java/org/apache/paimon/partition/PartitionValuesTimeExpireStrategy.java
+++
b/paimon-core/src/main/java/org/apache/paimon/partition/PartitionValuesTimeExpireStrategy.java
@@ -81,23 +81,34 @@ public class PartitionValuesTimeExpireStrategy extends
PartitionExpireStrategy {
LocalDateTime partTime = timeExtractor.extract(partitionKeys,
Arrays.asList(array));
return expireDateTime.isAfter(partTime);
} catch (DateTimeParseException e) {
- String partitionInfo =
- IntStream.range(0, partitionKeys.size())
- .mapToObj(i -> partitionKeys.get(i) + ":" +
array[i])
- .collect(Collectors.joining(","));
LOG.warn(
"Can't extract datetime from partition {}. If you want
to configure partition expiration, please:\n"
+ " 1. Check the expiration configuration.\n"
+ " 2. Manually delete the partition using
the drop-partition command if the partition"
+ " value is non-date formatted.\n"
+ " 3. Use '{}' expiration strategy by set
'{}', which supports non-date formatted partition.",
- partitionInfo,
+ formatPartitionInfo(array),
+ CoreOptions.PartitionExpireStrategy.UPDATE_TIME,
+ CoreOptions.PARTITION_EXPIRATION_STRATEGY.key());
+ return false;
+ } catch (NullPointerException e) {
+ // there might exist NULL partition value
+ LOG.warn(
+ "This partition {} cannot be expired because it
contains null value. "
+ + "You can try to drop it manually or use '{}'
expiration strategy by set '{}'.",
+ formatPartitionInfo(array),
CoreOptions.PartitionExpireStrategy.UPDATE_TIME,
CoreOptions.PARTITION_EXPIRATION_STRATEGY.key());
return false;
}
}
+ private String formatPartitionInfo(Object[] array) {
+ return IntStream.range(0, partitionKeys.size())
+ .mapToObj(i -> partitionKeys.get(i) + ":" + array[i])
+ .collect(Collectors.joining(","));
+ }
+
@Override
public boolean test(
long rowCount,
diff --git
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/ExpirePartitionsProcedureITCase.java
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/ExpirePartitionsProcedureITCase.java
index bc2e84902f..2d1fb6dde7 100644
---
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/ExpirePartitionsProcedureITCase.java
+++
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/ExpirePartitionsProcedureITCase.java
@@ -402,6 +402,19 @@ public class ExpirePartitionsProcedureITCase extends
CatalogITCaseBase {
.containsExactlyInAnyOrder("4:2024-06-03:01:00",
"Never-expire:9999-09-09:99:99");
}
+ @Test
+ public void testNullPartitionExpire() {
+ sql("CREATE TABLE T (k INT, ds STRING) PARTITIONED BY (ds);");
+ sql("INSERT INTO T VALUES (1, CAST (NULL AS STRING))");
+ assertThat(
+ callExpirePartitions(
+ "CALL sys.expire_partitions("
+ + "`table` => 'default.T'"
+ + ", expiration_time => '1 d'"
+ + ", timestamp_formatter =>
'yyyyMMdd')"))
+ .containsExactly("No expired partitions.");
+ }
+
/** Return a list of expired partitions. */
public List<String> callExpirePartitions(String callSql) {
return sql(callSql).stream()
