This is an automated email from the ASF dual-hosted git repository.
leaves12138 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 3cd37e7131 [core] Fix format table date partition pruning (#8011)
3cd37e7131 is described below
commit 3cd37e7131a701a1fbaf0dffdb7e2691951f676c
Author: YeJunHao <[email protected]>
AuthorDate: Thu May 28 17:47:01 2026 +0800
[core] Fix format table date partition pruning (#8011)
---
.../paimon/table/format/FormatTableScan.java | 27 ++++++++++++++--
.../paimon/table/format/FormatTableScanTest.java | 37 ++++++++++++++++++----
2 files changed, 55 insertions(+), 9 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableScan.java
b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableScan.java
index 9a561a6bd1..9bbd64ccdf 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableScan.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableScan.java
@@ -19,7 +19,10 @@
package org.apache.paimon.table.format;
import org.apache.paimon.CoreOptions;
+import org.apache.paimon.casting.CastExecutor;
+import org.apache.paimon.casting.CastExecutors;
import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.data.BinaryString;
import org.apache.paimon.data.GenericRow;
import org.apache.paimon.data.serializer.InternalRowSerializer;
import org.apache.paimon.format.csv.CsvOptions;
@@ -43,7 +46,9 @@ import
org.apache.paimon.table.format.predicate.PredicateUtils;
import org.apache.paimon.table.source.InnerTableScan;
import org.apache.paimon.table.source.Split;
import org.apache.paimon.table.source.TableScan;
+import org.apache.paimon.types.DataType;
import org.apache.paimon.types.RowType;
+import org.apache.paimon.types.VarCharType;
import org.apache.paimon.utils.InternalRowPartitionComputer;
import org.apache.paimon.utils.Pair;
import org.apache.paimon.utils.PartitionPathUtils;
@@ -245,7 +250,8 @@ public class FormatTableScan implements InnerTableScan {
Map<String, String> equalityPrefix =
extractLeadingEqualityPartitionSpecWhenOnlyAnd(
partitionKeys,
- ((DefaultPartitionPredicate)
partitionFilter).predicate());
+ ((DefaultPartitionPredicate)
partitionFilter).predicate(),
+ partitionType);
if (!equalityPrefix.isEmpty()) {
// Use optimized scan for specific partition path
String partitionPath =
@@ -313,7 +319,7 @@ public class FormatTableScan implements InnerTableScan {
}
public static Map<String, String>
extractLeadingEqualityPartitionSpecWhenOnlyAnd(
- List<String> partitionKeys, Predicate predicate) {
+ List<String> partitionKeys, Predicate predicate, RowType
partitionType) {
List<Predicate> predicates = PredicateBuilder.splitAnd(predicate);
Map<String, String> equals = new HashMap<>();
for (Predicate sub : predicates) {
@@ -324,7 +330,10 @@ public class FormatTableScan implements InnerTableScan {
LeafFunction function = ((LeafPredicate) sub).function();
String field = fieldRef.name();
if (function instanceof Equal &&
partitionKeys.contains(field)) {
- equals.put(field, ((LeafPredicate)
sub).literals().get(0).toString());
+ equals.put(
+ field,
+ partitionLiteralToString(
+ fieldRef.type(), ((LeafPredicate)
sub).literals().get(0)));
}
}
}
@@ -339,4 +348,16 @@ public class FormatTableScan implements InnerTableScan {
}
return result;
}
+
+ private static String partitionLiteralToString(DataType type, Object
literal) {
+ if (literal == null) {
+ return null;
+ }
+
+ CastExecutor<Object, BinaryString> executor =
+ (CastExecutor<Object, BinaryString>)
+ CastExecutors.resolve(type, VarCharType.STRING_TYPE);
+ BinaryString value = executor.cast(literal);
+ return value == null ? null : value.toString();
+ }
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableScanTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableScanTest.java
index e6602fb716..cda08b61e7 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableScanTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableScanTest.java
@@ -237,6 +237,31 @@ public class FormatTableScanTest {
assertThat(searched.size()).isEqualTo(1);
}
+ @TestTemplate
+ void testComputeScanPathWithDateEqualityFilter() {
+ Path tableLocation = new Path(tmpPath.toUri());
+ RowType datePartitionType = RowType.builder().field("dt",
DataTypes.DATE()).build();
+ List<String> datePartitionKeys = datePartitionType.getFieldNames();
+
+ PredicateBuilder builder = new PredicateBuilder(datePartitionType);
+ Predicate equalityPredicate =
+ builder.equal(0, (int)
java.time.LocalDate.parse("2026-05-01").toEpochDay());
+ PartitionPredicate partitionFilter =
+ PartitionPredicate.fromPredicate(datePartitionType,
equalityPredicate);
+
+ Pair<Path, Integer> result =
+ FormatTableScan.computeScanPathAndLevel(
+ tableLocation,
+ datePartitionKeys,
+ partitionFilter,
+ datePartitionType,
+ enablePartitionValueOnly);
+ String partitionPath = enablePartitionValueOnly ? "2026-05-01" :
"dt=2026-05-01";
+
+ assertThat(result.getLeft().toString()).isEqualTo(tableLocation +
partitionPath);
+ assertThat(result.getRight()).isEqualTo(0);
+ }
+
@TestTemplate
void testComputeScanPathWithFirstLevel() throws IOException {
Path tableLocation = new Path(tmpPath.toUri());
@@ -554,7 +579,7 @@ public class FormatTableScanTest {
Map<String, String> result =
FormatTableScan.extractLeadingEqualityPartitionSpecWhenOnlyAnd(
- partitionKeys, equalityPredicate);
+ partitionKeys, equalityPredicate, type);
assertThat(result).hasSize(3);
assertThat(result.get("year")).isEqualTo("2023");
@@ -581,7 +606,7 @@ public class FormatTableScanTest {
Map<String, String> result =
FormatTableScan.extractLeadingEqualityPartitionSpecWhenOnlyAnd(
- partitionKeys, mixedPredicate);
+ partitionKeys, mixedPredicate, type);
assertThat(result).isNotNull();
assertThat(result).hasSize(2);
@@ -609,7 +634,7 @@ public class FormatTableScanTest {
Map<String, String> result =
FormatTableScan.extractLeadingEqualityPartitionSpecWhenOnlyAnd(
- partitionKeys, mixedPredicate);
+ partitionKeys, mixedPredicate, type);
assertThat(result).hasSize(1);
assertThat(result.get("year")).isEqualTo("2023");
assertThat(result.containsKey("month")).isFalse();
@@ -635,7 +660,7 @@ public class FormatTableScanTest {
Map<String, String> result =
FormatTableScan.extractLeadingEqualityPartitionSpecWhenOnlyAnd(
- partitionKeys, mixedPredicate);
+ partitionKeys, mixedPredicate, type);
assertThat(result).isEmpty();
}
@@ -656,7 +681,7 @@ public class FormatTableScanTest {
Map<String, String> result =
FormatTableScan.extractLeadingEqualityPartitionSpecWhenOnlyAnd(
- partitionKeys, nonEqualityPredicate);
+ partitionKeys, nonEqualityPredicate, type);
assertThat(result).isEmpty();
}
@@ -676,7 +701,7 @@ public class FormatTableScanTest {
Map<String, String> result =
FormatTableScan.extractLeadingEqualityPartitionSpecWhenOnlyAnd(
- partitionKeys, orPredicate);
+ partitionKeys, orPredicate, type);
assertThat(result).isEmpty();
}
