This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 1b7f43d9fca [fix][paimon] Fix Paimon date partition conversion failure
(#54043)
1b7f43d9fca is described below
commit 1b7f43d9fca6c01542cf43336155e4a5d2c0b606
Author: Petrichor <[email protected]>
AuthorDate: Mon Aug 4 06:03:05 2025 +0800
[fix][paimon] Fix Paimon date partition conversion failure (#54043)
### What problem does this PR solve?
**Background**: PR [#46641](https://github.com/apache/doris/pull/46641)
worked around Paimon date partition failures by treating date partitions
as unpartitioned tables.
**Root Cause**: Paimon stores date partition values as integers (e.g.,
`20231225` for `2023-12-25`), but the previous implementation failed to
convert these to Doris-compatible date formats, resulting in:
```
org.apache.doris.common.AnalysisException: errCode = 2, detailMessage =
date literal [19737] is invalid: null
```
---
.../apache/doris/datasource/paimon/PaimonUtil.java | 21 +++++++++++++++------
regression-test/data/mtmv_p0/test_paimon_mtmv.out | Bin 1353 -> 1394 bytes
.../suites/mtmv_p0/test_paimon_mtmv.groovy | 14 ++++++++++----
3 files changed, 25 insertions(+), 10 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
index a0ede02a32a..8f82f3c3aa0 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
@@ -59,6 +59,7 @@ import org.apache.paimon.types.DecimalType;
import org.apache.paimon.types.MapType;
import org.apache.paimon.types.RowType;
import org.apache.paimon.types.VarCharType;
+import org.apache.paimon.utils.DateTimeUtils;
import org.apache.paimon.utils.InstantiationUtil;
import org.apache.paimon.utils.Pair;
import org.apache.paimon.utils.Projection;
@@ -116,12 +117,23 @@ public class PaimonUtil {
Map<String, PartitionItem> nameToPartitionItem = Maps.newHashMap();
Map<String, Partition> nameToPartition = Maps.newHashMap();
PaimonPartitionInfo partitionInfo = new
PaimonPartitionInfo(nameToPartitionItem, nameToPartition);
+ List<Type> types = partitionColumns.stream()
+ .map(Column::getType)
+ .collect(Collectors.toList());
+ Map<String, Type> columnNameToType = partitionColumns.stream()
+ .collect(Collectors.toMap(Column::getName, Column::getType));
for (Partition partition : paimonPartitions) {
Map<String, String> spec = partition.spec();
StringBuilder sb = new StringBuilder();
for (Map.Entry<String, String> entry : spec.entrySet()) {
-
sb.append(entry.getKey()).append("=").append(entry.getValue()).append("/");
+ sb.append(entry.getKey()).append("=");
+ // Paimon stores DATE type as days since 1970-01-01 (epoch),
so we convert the integer to a date string.
+ if (columnNameToType.getOrDefault(entry.getKey(),
Type.NULL).isDateV2()) {
+
sb.append(DateTimeUtils.formatDate(Integer.parseInt(entry.getValue()))).append("/");
+ } else {
+ sb.append(entry.getValue()).append("/");
+ }
}
if (sb.length() > 0) {
sb.deleteCharAt(sb.length() - 1);
@@ -131,7 +143,7 @@ public class PaimonUtil {
try {
// partition values return by paimon api, may have problem,
// to avoid affecting the query, we catch exceptions here
- nameToPartitionItem.put(partitionName,
toListPartitionItem(partitionName, partitionColumns));
+ nameToPartitionItem.put(partitionName,
toListPartitionItem(partitionName, types));
} catch (Exception e) {
LOG.warn("toListPartitionItem failed, partitionColumns: {},
partitionValues: {}",
partitionColumns, partition.spec(), e);
@@ -140,11 +152,8 @@ public class PaimonUtil {
return partitionInfo;
}
- public static ListPartitionItem toListPartitionItem(String partitionName,
List<Column> partitionColumns)
+ public static ListPartitionItem toListPartitionItem(String partitionName,
List<Type> types)
throws AnalysisException {
- List<Type> types = partitionColumns.stream()
- .map(Column::getType)
- .collect(Collectors.toList());
// Partition name will be in format: nation=cn/city=beijing
// parse it to get values "cn" and "beijing"
List<String> partitionValues =
HiveUtil.toPartitionValues(partitionName);
diff --git a/regression-test/data/mtmv_p0/test_paimon_mtmv.out
b/regression-test/data/mtmv_p0/test_paimon_mtmv.out
index 0cfe4bd293c..5c7547c0687 100644
Binary files a/regression-test/data/mtmv_p0/test_paimon_mtmv.out and
b/regression-test/data/mtmv_p0/test_paimon_mtmv.out differ
diff --git a/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy
b/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy
index abd0f64450e..c4ea889f923 100644
--- a/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy
@@ -268,8 +268,7 @@ suite("test_paimon_mtmv",
"p0,external,mtmv,external_docker,external_docker_dori
// date type will has problem
order_qt_date_partition_base_table "SELECT * FROM
${catalogName}.`test_paimon_spark`.date_partition"
- test {
- sql """
+ sql """
CREATE MATERIALIZED VIEW ${mvName}
BUILD DEFERRED REFRESH AUTO ON MANUAL
partition by (`create_date`)
@@ -278,8 +277,15 @@ suite("test_paimon_mtmv",
"p0,external,mtmv,external_docker,external_docker_dori
AS
SELECT * FROM
${catalogName}.`test_paimon_spark`.date_partition;
"""
- exception "Unable to find a suitable base table"
- }
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName} auto;
+ """
+ waitingMTMVTaskFinishedByMvName(mvName)
+ def showPaimonDateTypePartitionsResult = sql """show partitions from
${mvName}"""
+ logger.info("showPaimonDateTypePartitionsResult: " +
showPaimonDateTypePartitionsResult.toString())
+
assertTrue(showPaimonDateTypePartitionsResult.toString().contains("p_20200101"))
+ order_qt_date_type_partition "select * FROM ${mvName}"
+ sql """drop materialized view if exists ${mvName};"""
sql """
CREATE MATERIALIZED VIEW ${mvName}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
