This is an automated email from the ASF dual-hosted git repository.
zhangstar333 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new f895ef03c7a [opt](paimon) support paimon table with
partition.legacy-name in date type (#61076)
f895ef03c7a is described below
commit f895ef03c7acd97fbaa5853bd4983c0c0c05647b
Author: zhangstar333 <[email protected]>
AuthorDate: Tue Mar 10 11:08:58 2026 +0800
[opt](paimon) support paimon table with partition.legacy-name in date type
(#61076)
### What problem does this PR solve?
Problem Summary:
```
mysql> select * from test_partition_legacy_false;
ERROR 1105 (HY000): errCode = 2, detailMessage = failed to load paimon
snapshot 1772691627468.test_partition_legacy.test_partition_legacy_false: For
input string: "2026-03-01"
mysql>
```
when partition.legacy-name = false, the partition date value is
"2026-03-01" not 20513, so can't parse it as int value
---
.../create_preinstalled_scripts/paimon/run12.sql | 54 ++++++++++++++++++++
.../datasource/paimon/PaimonMetadataCache.java | 8 +--
.../apache/doris/datasource/paimon/PaimonUtil.java | 21 ++++++--
.../paimon/paimon_partition_legacy.out | 41 +++++++++++++++
.../paimon/paimon_partition_legacy.groovy | 59 ++++++++++++++++++++++
5 files changed, 176 insertions(+), 7 deletions(-)
diff --git
a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run12.sql
b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run12.sql
new file mode 100644
index 00000000000..c7b26a6b4c6
--- /dev/null
+++
b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run12.sql
@@ -0,0 +1,54 @@
+use paimon;
+create database if not exists test_partition_legacy;
+use test_partition_legacy;
+
+drop table if exists test_partition_legacy_true;
+CREATE TABLE test_partition_legacy_true (
+ dt DATE,
+ user_id BIGINT,
+ event_name STRING,
+ event_value DOUBLE
+) USING paimon
+PARTITIONED BY (dt)
+TBLPROPERTIES (
+ 'primary-key' = 'dt, user_id',
+ 'bucket' = '1',
+ 'merge-engine' = 'deduplicate',
+ 'partition.legacy-name' = 'true'
+);
+
+INSERT INTO test_partition_legacy_true (dt, user_id, event_name, event_value)
VALUES
+ (CAST('2026-02-13' AS DATE), CAST(1001 AS BIGINT), 'click', CAST(1.5 AS
DOUBLE)),
+ (CAST('2026-02-13' AS DATE), CAST(1002 AS BIGINT), 'view', CAST(2.0 AS
DOUBLE)),
+ (CAST('2026-02-13' AS DATE), CAST(1003 AS BIGINT), 'purchase', CAST(99.9
AS DOUBLE)),
+ (CAST('2026-03-01' AS DATE), CAST(2001 AS BIGINT), 'click', CAST(3.0 AS
DOUBLE)),
+ (CAST('2026-03-01' AS DATE), CAST(2002 AS BIGINT), 'view', CAST(4.5 AS
DOUBLE)),
+ (CAST('2026-03-02' AS DATE), CAST(3001 AS BIGINT), 'click', CAST(5.0 AS
DOUBLE)),
+ (CAST('2026-03-02' AS DATE), CAST(3002 AS BIGINT), 'purchase', CAST(188.0
AS DOUBLE)),
+ (CAST('2026-03-02' AS DATE), CAST(3003 AS BIGINT), 'view', CAST(6.5 AS
DOUBLE));
+
+
+drop table if exists test_partition_legacy_false;
+CREATE TABLE test_partition_legacy_false (
+ dt DATE,
+ user_id BIGINT,
+ event_name STRING,
+ event_value DOUBLE
+) USING paimon
+PARTITIONED BY (dt)
+TBLPROPERTIES (
+ 'primary-key' = 'dt, user_id',
+ 'bucket' = '1',
+ 'merge-engine' = 'deduplicate',
+ 'partition.legacy-name' = 'false'
+);
+
+INSERT INTO test_partition_legacy_false (dt, user_id, event_name, event_value)
VALUES
+ (CAST('2026-02-13' AS DATE), CAST(1001 AS BIGINT), 'click', CAST(1.5 AS
DOUBLE)),
+ (CAST('2026-02-13' AS DATE), CAST(1002 AS BIGINT), 'view', CAST(2.0 AS
DOUBLE)),
+ (CAST('2026-02-13' AS DATE), CAST(1003 AS BIGINT), 'purchase', CAST(99.9
AS DOUBLE)),
+ (CAST('2026-03-01' AS DATE), CAST(2001 AS BIGINT), 'click', CAST(3.0 AS
DOUBLE)),
+ (CAST('2026-03-01' AS DATE), CAST(2002 AS BIGINT), 'view', CAST(4.5 AS
DOUBLE)),
+ (CAST('2026-03-02' AS DATE), CAST(3001 AS BIGINT), 'click', CAST(5.0 AS
DOUBLE)),
+ (CAST('2026-03-02' AS DATE), CAST(3002 AS BIGINT), 'purchase', CAST(188.0
AS DOUBLE)),
+ (CAST('2026-03-02' AS DATE), CAST(3003 AS BIGINT), 'view', CAST(6.5 AS
DOUBLE));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataCache.java
index 7f118490fdd..222e5b19ea7 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataCache.java
@@ -105,7 +105,8 @@ public class PaimonMetadataCache {
PaimonSnapshot latestSnapshot = loadLatestSnapshot(paimonTable,
nameMapping);
List<Column> partitionColumns =
getPaimonSchemaCacheValue(nameMapping,
latestSnapshot.getSchemaId()).getPartitionColumns();
- PaimonPartitionInfo partitionInfo = loadPartitionInfo(nameMapping,
partitionColumns);
+ boolean legacyPartitionName =
PaimonUtil.isLegacyPartitionName(paimonTable);
+ PaimonPartitionInfo partitionInfo = loadPartitionInfo(nameMapping,
partitionColumns, legacyPartitionName);
return new PaimonSnapshotCacheValue(partitionInfo, latestSnapshot);
} catch (Exception e) {
throw new CacheException("failed to load paimon snapshot %s.%s.%s:
%s",
@@ -132,7 +133,8 @@ public class PaimonMetadataCache {
return (PaimonSchemaCacheValue) schemaCacheValue.get();
}
- private PaimonPartitionInfo loadPartitionInfo(NameMapping nameMapping,
List<Column> partitionColumns)
+ private PaimonPartitionInfo loadPartitionInfo(NameMapping nameMapping,
List<Column> partitionColumns,
+ boolean legacyPartitionName)
throws AnalysisException {
if (CollectionUtils.isEmpty(partitionColumns)) {
return PaimonPartitionInfo.EMPTY;
@@ -140,7 +142,7 @@ public class PaimonMetadataCache {
PaimonExternalCatalog externalCatalog = (PaimonExternalCatalog)
Env.getCurrentEnv().getCatalogMgr()
.getCatalogOrAnalysisException(nameMapping.getCtlId());
List<Partition> paimonPartitions =
externalCatalog.getPaimonPartitions(nameMapping);
- return PaimonUtil.generatePartitionInfo(partitionColumns,
paimonPartitions);
+ return PaimonUtil.generatePartitionInfo(partitionColumns,
paimonPartitions, legacyPartitionName);
}
private PaimonSnapshot loadLatestSnapshot(Table paimonTable, NameMapping
nameMapping) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
index d4c9ddf6c2f..a80da01cf96 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
@@ -105,11 +105,20 @@ public class PaimonUtil {
private static final String SYS_TABLE_TYPE_AUDIT_LOG = "audit_log";
private static final String SYS_TABLE_TYPE_BINLOG = "binlog";
private static final String TABLE_READ_SEQUENCE_NUMBER_ENABLED =
"table-read.sequence-number.enabled";
+ private static final String PARTITION_LEGACY_NAME =
"partition.legacy-name";
public static boolean isDigitalString(String value) {
return value != null && DIGITAL_REGEX.matcher(value).matches();
}
+ /**
+ * Extract the legacy partition name configuration from Paimon table
options.
+ */
+ public static boolean isLegacyPartitionName(Table paimonTable) {
+ return Boolean.parseBoolean(
+ paimonTable.options().getOrDefault(PARTITION_LEGACY_NAME,
"true"));
+ }
+
public static List<InternalRow> read(
Table table, @Nullable int[] projection, @Nullable Predicate
predicate,
Pair<ConfigOption<?>, String>... dynamicOptions)
@@ -141,7 +150,7 @@ public class PaimonUtil {
}
public static PaimonPartitionInfo generatePartitionInfo(List<Column>
partitionColumns,
- List<Partition> paimonPartitions) {
+ List<Partition> paimonPartitions, boolean legacyPartitionName) {
if (CollectionUtils.isEmpty(partitionColumns) ||
paimonPartitions.isEmpty()) {
return PaimonPartitionInfo.EMPTY;
@@ -161,8 +170,11 @@ public class PaimonUtil {
StringBuilder sb = new StringBuilder();
for (Map.Entry<String, String> entry : spec.entrySet()) {
sb.append(entry.getKey()).append("=");
- // Paimon stores DATE type as days since 1970-01-01 (epoch),
so we convert the integer to a date string.
- if (columnNameToType.getOrDefault(entry.getKey(),
Type.NULL).isDateV2()) {
+ // When partition.legacy-name = true (default), Paimon stores
DATE type as days since
+ // 1970-01-01 (epoch integer), so we need to convert the
integer to a date string.
+ // When partition.legacy-name = false, the value is already a
human read date string.
+ if (legacyPartitionName
+ && columnNameToType.getOrDefault(entry.getKey(),
Type.NULL).isDateV2()) {
sb.append(DateTimeUtils.formatDate(Integer.parseInt(entry.getValue()))).append("/");
} else {
sb.append(entry.getValue()).append("/");
@@ -565,7 +577,8 @@ public class PaimonUtil {
return null;
}
return value.toString();
- // case binary, varbinary should not supported, because if return
string with utf8,
+ // case binary:
+ // case varbinary: should not supported, because if return string
with utf8,
// the data maybe be corrupted
case DATE:
if (value == null) {
diff --git
a/regression-test/data/external_table_p0/paimon/paimon_partition_legacy.out
b/regression-test/data/external_table_p0/paimon/paimon_partition_legacy.out
new file mode 100644
index 00000000000..101b6d3e243
--- /dev/null
+++ b/regression-test/data/external_table_p0/paimon/paimon_partition_legacy.out
@@ -0,0 +1,41 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !order --
+2026-02-13 1001 click 1.5
+2026-02-13 1002 view 2
+2026-03-01 2001 click 3
+2026-03-01 2002 view 4.5
+2026-03-02 3001 click 5
+2026-03-02 3003 view 6.5
+2026-02-13 1003 purchase 99.90000000000001
+2026-03-02 3002 purchase 188
+
+-- !order --
+2026-02-13 1001 click 1.5
+2026-02-13 1002 view 2
+2026-03-01 2001 click 3
+2026-03-01 2002 view 4.5
+2026-03-02 3001 click 5
+2026-03-02 3003 view 6.5
+2026-02-13 1003 purchase 99.90000000000001
+2026-03-02 3002 purchase 188
+
+-- !order --
+2026-02-13 1001 click 1.5
+2026-02-13 1002 view 2
+2026-03-01 2001 click 3
+2026-03-01 2002 view 4.5
+2026-03-02 3001 click 5
+2026-03-02 3003 view 6.5
+2026-02-13 1003 purchase 99.90000000000001
+2026-03-02 3002 purchase 188
+
+-- !order --
+2026-02-13 1001 click 1.5
+2026-02-13 1002 view 2
+2026-03-01 2001 click 3
+2026-03-01 2002 view 4.5
+2026-03-02 3001 click 5
+2026-03-02 3003 view 6.5
+2026-02-13 1003 purchase 99.90000000000001
+2026-03-02 3002 purchase 188
+
diff --git
a/regression-test/suites/external_table_p0/paimon/paimon_partition_legacy.groovy
b/regression-test/suites/external_table_p0/paimon/paimon_partition_legacy.groovy
new file mode 100644
index 00000000000..0dca3a7ac41
--- /dev/null
+++
b/regression-test/suites/external_table_p0/paimon/paimon_partition_legacy.groovy
@@ -0,0 +1,59 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("paimon_partition_legacy", "p0,external") {
+
+ logger.info("start paimon test")
+ String enabled = context.config.otherConfigs.get("enablePaimonTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("disabled paimon test")
+ return
+ }
+
+ try {
+ String catalog_name = "test_partition_legacy_catalog"
+ String minio_port =
context.config.otherConfigs.get("iceberg_minio_port")
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+
+ sql """drop catalog if exists ${catalog_name}"""
+ sql """CREATE CATALOG ${catalog_name} PROPERTIES (
+ 'type'='paimon',
+ 'warehouse' = 's3://warehouse/wh/',
+ "s3.access_key" = "admin",
+ "s3.secret_key" = "password",
+ "s3.endpoint" = "http://${externalEnvIp}:${minio_port}";,
+ "s3.region" = "us-east-1"
+ );"""
+
+ logger.info("catalog " + catalog_name + " created")
+ sql """switch ${catalog_name};"""
+ logger.info("switched to catalog " + catalog_name)
+ sql """use test_partition_legacy;"""
+
+ sql """set force_jni_scanner=true"""
+ qt_order """ select * from test_partition_legacy_true order by
event_value; """
+ qt_order """ select * from test_partition_legacy_false order by
event_value; """
+
+ sql """set force_jni_scanner=false"""
+ qt_order """ select * from test_partition_legacy_true order by
event_value; """
+ qt_order """ select * from test_partition_legacy_false order by
event_value; """
+
+ } finally {
+ sql """set force_jni_scanner=false"""
+ }
+
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
