This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 7b35dcf7fdd branch-4.1: [fix](paimon) Handle special characters in
partition values #65904 (#65938)
7b35dcf7fdd is described below
commit 7b35dcf7fdd49d131d640d1f0a4d05591153e1a2
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Thu Jul 23 20:19:02 2026 +0800
branch-4.1: [fix](paimon) Handle special characters in partition values
#65904 (#65938)
Cherry-picked from #65904
Co-authored-by: zhangstar333 <[email protected]>
---
.../apache/doris/datasource/paimon/PaimonUtil.java | 48 ++++----
.../doris/datasource/paimon/PaimonUtilTest.java | 123 +++++++++++++++++++++
2 files changed, 149 insertions(+), 22 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
index 1c71ac3dd24..ea49e80a5b9 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
@@ -29,7 +29,6 @@ import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.datasource.ExternalTable;
-import org.apache.doris.datasource.hive.HiveUtil;
import org.apache.doris.thrift.TColumnType;
import org.apache.doris.thrift.TPrimitiveType;
import org.apache.doris.thrift.schema.external.TArrayField;
@@ -77,6 +76,7 @@ import org.apache.paimon.types.VarCharType;
import org.apache.paimon.utils.DateTimeUtils;
import org.apache.paimon.utils.InstantiationUtil;
import org.apache.paimon.utils.Pair;
+import org.apache.paimon.utils.PartitionPathUtils;
import org.apache.paimon.utils.Projection;
import org.apache.paimon.utils.RowDataToObjectArrayConverter;
@@ -91,6 +91,7 @@ import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Base64;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -162,47 +163,50 @@ public class PaimonUtil {
List<Type> types = partitionColumns.stream()
.map(Column::getType)
.collect(Collectors.toList());
- Map<String, Type> columnNameToType = partitionColumns.stream()
- .collect(Collectors.toMap(Column::getName, Column::getType));
for (Partition partition : paimonPartitions) {
Map<String, String> spec = partition.spec();
- StringBuilder sb = new StringBuilder();
- for (Map.Entry<String, String> entry : spec.entrySet()) {
- sb.append(entry.getKey()).append("=");
+ // Paimon partition specs contain logical values, which may
include path separators.
+ // Build partition values directly instead of parsing them as a
Hive partition path.
+ List<String> partitionValues =
Lists.newArrayListWithExpectedSize(partitionColumns.size());
+ LinkedHashMap<String, String> orderedPartitionSpec = new
LinkedHashMap<>();
+ for (Column partitionColumn : partitionColumns) {
+ String partitionColumnName = partitionColumn.getName();
+ String partitionValue = spec.get(partitionColumnName);
// When partition.legacy-name = true (default), Paimon stores
DATE type as days since
// 1970-01-01 (epoch integer), so we need to convert the
integer to a date string.
// When partition.legacy-name = false, the value is already a
human read date string.
- if (legacyPartitionName
- && columnNameToType.getOrDefault(entry.getKey(),
Type.NULL).isDateV2()) {
-
sb.append(DateTimeUtils.formatDate(Integer.parseInt(entry.getValue()))).append("/");
- } else {
- sb.append(entry.getValue()).append("/");
+ if (legacyPartitionName &&
partitionColumn.getType().isDateV2()) {
+ partitionValue =
DateTimeUtils.formatDate(Integer.parseInt(partitionValue));
}
+ partitionValues.add(partitionValue);
+ orderedPartitionSpec.put(partitionColumnName, partitionValue);
}
- if (sb.length() > 0) {
- sb.deleteCharAt(sb.length() - 1);
- }
- String partitionName = sb.toString();
- nameToPartition.put(partitionName, partition);
+ String partitionPath =
PartitionPathUtils.generatePartitionPath(orderedPartitionSpec);
+ String partitionName = partitionPath.substring(0,
partitionPath.length() - 1);
+ Partition previousPartition =
nameToPartition.putIfAbsent(partitionName, partition);
+ Preconditions.checkState(previousPartition == null,
+ "Duplicate Paimon partition name: " + partitionName);
+ PartitionItem partitionItem;
try {
// partition values return by paimon api, may have problem,
// to avoid affecting the query, we catch exceptions here
- nameToPartitionItem.put(partitionName,
toListPartitionItem(partitionName, types));
+ partitionItem = toListPartitionItem(partitionValues, types);
} catch (Exception e) {
LOG.warn("toListPartitionItem failed, partitionColumns: {},
partitionValues: {}",
partitionColumns, partition.spec(), e);
+ continue;
}
+ PartitionItem previousPartitionItem =
nameToPartitionItem.putIfAbsent(partitionName, partitionItem);
+ Preconditions.checkState(previousPartitionItem == null,
+ "Duplicate Paimon partition item name: " + partitionName);
}
return partitionInfo;
}
- public static ListPartitionItem toListPartitionItem(String partitionName,
List<Type> types)
+ public static ListPartitionItem toListPartitionItem(List<String>
partitionValues, List<Type> types)
throws AnalysisException {
- // Partition name will be in format: nation=cn/city=beijing
- // parse it to get values "cn" and "beijing"
- List<String> partitionValues =
HiveUtil.toPartitionValues(partitionName);
- Preconditions.checkState(partitionValues.size() == types.size(),
partitionName + " vs. " + types);
+ Preconditions.checkState(partitionValues.size() == types.size(),
partitionValues + " vs. " + types);
List<PartitionValue> values =
Lists.newArrayListWithExpectedSize(types.size());
for (String partitionValue : partitionValues) {
// null will in partition 'null'
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java
index 93cbba81cb7..fe4c94f2f1b 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java
@@ -18,6 +18,8 @@
package org.apache.doris.datasource.paimon;
import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.ListPartitionItem;
+import org.apache.doris.catalog.PartitionItem;
import org.apache.doris.catalog.Type;
import org.apache.doris.thrift.TPrimitiveType;
import org.apache.doris.thrift.schema.external.TFieldPtr;
@@ -26,6 +28,7 @@ import org.apache.doris.thrift.schema.external.TSchema;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.BinaryRowWriter;
import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.partition.Partition;
import org.apache.paimon.schema.TableSchema;
import org.apache.paimon.table.Table;
import org.apache.paimon.types.CharType;
@@ -39,6 +42,7 @@ import org.mockito.Mockito;
import java.util.Arrays;
import java.util.Collections;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -128,6 +132,125 @@ public class PaimonUtilTest {
Assert.assertEquals("2026-05-26", partitionInfoMap.get("Dt"));
}
+ @Test
+ public void testGeneratePartitionInfoWithSpecialCharacters() {
+ List<Column> partitionColumns = Arrays.asList(
+ new Column("source", Type.STRING),
+ new Column("part_str", Type.STRING),
+ new Column("pass", Type.STRING));
+ Map<String, String> spec = new LinkedHashMap<>();
+ spec.put("source", "dataset/team-a/segment-01");
+ spec.put("part_str", "/ymd=20260701/hour=[0-9][0-9]/*.jsonl");
+ spec.put("pass", "s1");
+ Partition partition = new Partition(spec, 1L, 1L, 1L, 1L, false);
+
+ PaimonPartitionInfo partitionInfo = PaimonUtil.generatePartitionInfo(
+ partitionColumns, Collections.singletonList(partition), false);
+
+ Assert.assertFalse(partitionInfo.isPartitionInvalid());
+ Assert.assertEquals(1, partitionInfo.getNameToPartition().size());
+ Assert.assertEquals(1, partitionInfo.getNameToPartitionItem().size());
+ String partitionName = "source=dataset%2Fteam-a%2Fsegment-01"
+ +
"/part_str=%2Fymd%3D20260701%2Fhour%3D%5B0-9%5D%5B0-9%5D%2F%2A.jsonl/pass=s1";
+
Assert.assertTrue(partitionInfo.getNameToPartition().containsKey(partitionName));
+ PartitionItem partitionItem =
partitionInfo.getNameToPartitionItem().values().iterator().next();
+ List<String> actualValues = ((ListPartitionItem)
partitionItem).getItems().get(0)
+ .getPartitionValuesAsStringList();
+ Assert.assertEquals(Arrays.asList(
+ "dataset/team-a/segment-01",
+ "/ymd=20260701/hour=[0-9][0-9]/*.jsonl",
+ "s1"), actualValues);
+ }
+
+ @Test
+ public void testGeneratePartitionInfoUsesPartitionColumnOrder() {
+ List<Column> partitionColumns = Arrays.asList(
+ new Column("source", Type.STRING),
+ new Column("part_str", Type.STRING),
+ new Column("pass", Type.STRING));
+ Map<String, String> spec = new LinkedHashMap<>();
+ spec.put("pass", "s1");
+ spec.put("part_str", "/ymd=20260721");
+ spec.put("source", "dataset/team-a/segment-01");
+ Partition partition = new Partition(spec, 1L, 1L, 1L, 1L, false);
+
+ PaimonPartitionInfo partitionInfo = PaimonUtil.generatePartitionInfo(
+ partitionColumns, Collections.singletonList(partition), false);
+
+ String partitionName = "source=dataset%2Fteam-a%2Fsegment-01"
+ + "/part_str=%2Fymd%3D20260721/pass=s1";
+
Assert.assertTrue(partitionInfo.getNameToPartition().containsKey(partitionName));
+ PartitionItem partitionItem =
partitionInfo.getNameToPartitionItem().get(partitionName);
+ List<String> actualValues = ((ListPartitionItem)
partitionItem).getItems().get(0)
+ .getPartitionValuesAsStringList();
+ Assert.assertEquals(Arrays.asList(
+ "dataset/team-a/segment-01", "/ymd=20260721", "s1"),
actualValues);
+ }
+
+ @Test
+ public void testGeneratePartitionInfoPreservesLegacyDateConversion() {
+ List<Column> partitionColumns = Collections.singletonList(new
Column("dt", Type.DATEV2));
+ Map<String, String> spec = new LinkedHashMap<>();
+ spec.put("dt", "19737");
+ Partition partition = new Partition(spec, 1L, 1L, 1L, 1L, false);
+
+ PaimonPartitionInfo partitionInfo = PaimonUtil.generatePartitionInfo(
+ partitionColumns, Collections.singletonList(partition), true);
+
+ String partitionName = "dt=2024-01-15";
+
Assert.assertTrue(partitionInfo.getNameToPartition().containsKey(partitionName));
+ PartitionItem partitionItem =
partitionInfo.getNameToPartitionItem().get(partitionName);
+ Assert.assertEquals(Collections.singletonList("2024-01-15"),
+ ((ListPartitionItem)
partitionItem).getItems().get(0).getPartitionValuesAsStringList());
+ }
+
+ @Test
+ public void testGeneratePartitionInfoUsesCollisionFreePartitionNames() {
+ List<Column> partitionColumns = Arrays.asList(
+ new Column("a", Type.STRING),
+ new Column("b", Type.STRING));
+ Map<String, String> firstSpec = new LinkedHashMap<>();
+ firstSpec.put("a", "x/b=y");
+ firstSpec.put("b", "z");
+ Map<String, String> secondSpec = new LinkedHashMap<>();
+ secondSpec.put("a", "x");
+ secondSpec.put("b", "y/b=z");
+ Partition firstPartition = new Partition(firstSpec, 1L, 1L, 1L, 1L,
false);
+ Partition secondPartition = new Partition(secondSpec, 2L, 2L, 2L, 2L,
false);
+
+ PaimonPartitionInfo partitionInfo = PaimonUtil.generatePartitionInfo(
+ partitionColumns, Arrays.asList(firstPartition,
secondPartition), false);
+
+ String firstPartitionName = "a=x%2Fb%3Dy/b=z";
+ String secondPartitionName = "a=x/b=y%2Fb%3Dz";
+ Assert.assertFalse(partitionInfo.isPartitionInvalid());
+ Assert.assertEquals(2, partitionInfo.getNameToPartition().size());
+ Assert.assertEquals(2, partitionInfo.getNameToPartitionItem().size());
+ Assert.assertSame(firstPartition,
partitionInfo.getNameToPartition().get(firstPartitionName));
+ Assert.assertSame(secondPartition,
partitionInfo.getNameToPartition().get(secondPartitionName));
+ Assert.assertEquals(Arrays.asList("x/b=y", "z"),
+ ((ListPartitionItem)
partitionInfo.getNameToPartitionItem().get(firstPartitionName))
+ .getItems().get(0).getPartitionValuesAsStringList());
+ Assert.assertEquals(Arrays.asList("x", "y/b=z"),
+ ((ListPartitionItem)
partitionInfo.getNameToPartitionItem().get(secondPartitionName))
+ .getItems().get(0).getPartitionValuesAsStringList());
+ }
+
+ @Test
+ public void testGeneratePartitionInfoRejectsDuplicatePartitionNames() {
+ List<Column> partitionColumns = Collections.singletonList(new
Column("part", Type.STRING));
+ Map<String, String> firstSpec = Collections.singletonMap("part",
"same");
+ Map<String, String> secondSpec = Collections.singletonMap("part",
"same");
+ Partition firstPartition = new Partition(firstSpec, 1L, 1L, 1L, 1L,
false);
+ Partition secondPartition = new Partition(secondSpec, 2L, 2L, 2L, 2L,
false);
+
+ IllegalStateException exception =
Assert.assertThrows(IllegalStateException.class,
+ () -> PaimonUtil.generatePartitionInfo(
+ partitionColumns, Arrays.asList(firstPartition,
secondPartition), false));
+
+ Assert.assertTrue(exception.getMessage().contains("Duplicate Paimon
partition name"));
+ }
+
@Test
public void testBinlogHistorySchemaWithSequenceNumber() {
PaimonSysExternalTable binlogTable =
Mockito.mock(PaimonSysExternalTable.class);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
