This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new a9f2d2c6d [core] Fix partition column generate wrong partition spec
(#4349)
a9f2d2c6d is described below
commit a9f2d2c6d70121e6fe7c9157f9627b5a0135b5e9
Author: Xiduo You <[email protected]>
AuthorDate: Wed Oct 23 21:58:32 2024 +0800
[core] Fix partition column generate wrong partition spec (#4349)
---
.../shortcodes/generated/core_configuration.html | 6 +++
.../main/java/org/apache/paimon/CoreOptions.java | 12 +++++
.../apache/paimon/casting/AbstractCastRule.java | 0
.../paimon/casting/BinaryToBinaryCastRule.java | 0
.../paimon/casting/BinaryToStringCastRule.java | 0
.../paimon/casting/BooleanToNumericCastRule.java | 0
.../paimon/casting/BooleanToStringCastRule.java | 0
.../org/apache/paimon/casting/CastExecutor.java | 0
.../org/apache/paimon/casting/CastExecutors.java | 0
.../org/apache/paimon/casting/CastFieldGetter.java | 0
.../java/org/apache/paimon/casting/CastRule.java | 0
.../apache/paimon/casting/CastRulePredicate.java | 0
.../java/org/apache/paimon/casting/CastedRow.java | 0
.../paimon/casting/DateToStringCastRule.java | 0
.../paimon/casting/DateToTimestampCastRule.java | 0
.../paimon/casting/DecimalToDecimalCastRule.java | 0
.../casting/DecimalToNumericPrimitiveCastRule.java | 0
.../org/apache/paimon/casting/DefaultValueRow.java | 0
.../paimon/casting/NumericPrimitiveCastRule.java | 0
.../casting/NumericPrimitiveToDecimalCastRule.java | 0
.../casting/NumericPrimitiveToTimestamp.java | 0
.../paimon/casting/NumericToBooleanCastRule.java | 0
.../paimon/casting/NumericToStringCastRule.java | 0
.../paimon/casting/StringToBinaryCastRule.java | 0
.../paimon/casting/StringToBooleanCastRule.java | 0
.../paimon/casting/StringToDateCastRule.java | 0
.../paimon/casting/StringToDecimalCastRule.java | 0
.../casting/StringToNumericPrimitiveCastRule.java | 0
.../paimon/casting/StringToStringCastRule.java | 0
.../paimon/casting/StringToTimeCastRule.java | 0
.../paimon/casting/StringToTimestampCastRule.java | 0
.../paimon/casting/TimeToStringCastRule.java | 0
.../paimon/casting/TimeToTimestampCastRule.java | 0
.../paimon/casting/TimestampToDateCastRule.java | 0
.../TimestampToNumericPrimitiveCastRule.java | 0
.../paimon/casting/TimestampToStringCastRule.java | 0
.../paimon/casting/TimestampToTimeCastRule.java | 0
.../casting/TimestampToTimestampCastRule.java | 0
.../paimon/utils/InternalRowPartitionComputer.java | 13 +++++-
.../java/org/apache/paimon/utils/TypeUtils.java | 20 ++++++++
.../apache/paimon/casting/CastExecutorTest.java | 0
.../java/org/apache/paimon/AbstractFileStore.java | 3 +-
.../java/org/apache/paimon/KeyValueFileStore.java | 3 +-
.../paimon/operation/AbstractFileStoreWrite.java | 8 +++-
.../paimon/operation/MemoryFileStoreWrite.java | 3 +-
.../apache/paimon/utils/FileStorePathFactory.java | 11 +++--
.../paimon/io/KeyValueFileReadWriteTest.java | 6 ++-
.../paimon/manifest/ManifestFileMetaTestBase.java | 3 +-
.../apache/paimon/manifest/ManifestFileTest.java | 3 +-
.../apache/paimon/manifest/ManifestListTest.java | 3 +-
.../paimon/utils/FileStorePathFactoryTest.java | 6 ++-
.../java/org/apache/paimon/flink/FlinkCatalog.java | 5 +-
.../flink/sink/partition/PartitionMarkDone.java | 3 +-
.../flink/source/TestChangelogDataReadWrite.java | 3 +-
.../apache/paimon/hive/HiveMetastoreClient.java | 6 ++-
.../paimon/spark/PaimonPartitionManagement.scala | 5 +-
.../commands/DeleteFromPaimonTableCommand.scala | 3 +-
.../apache/paimon/spark/sources/StreamHelper.scala | 9 ++--
.../apache/paimon/spark/SparkFileIndexITCase.java | 3 +-
.../org/apache/paimon/spark/sql/DDLTestBase.scala | 53 +++++++++++++++++++++-
60 files changed, 159 insertions(+), 31 deletions(-)
diff --git a/docs/layouts/shortcodes/generated/core_configuration.html
b/docs/layouts/shortcodes/generated/core_configuration.html
index fd788301e..9bbed03bd 100644
--- a/docs/layouts/shortcodes/generated/core_configuration.html
+++ b/docs/layouts/shortcodes/generated/core_configuration.html
@@ -575,6 +575,12 @@ This config option does not affect the default filesystem
metastore.</td>
<td>Duration</td>
<td>The expiration interval of a partition. A partition will be
expired if it‘s lifetime is over this value. Partition time is extracted from
the partition value.</td>
</tr>
+ <tr>
+ <td><h5>partition.legacy-name</h5></td>
+ <td style="word-wrap: break-word;">true</td>
+ <td>Boolean</td>
+ <td>The legacy partition name is using `toString` fpr all types.
If false, using cast to string for all types.</td>
+ </tr>
<tr>
<td><h5>partition.mark-done-action</h5></td>
<td style="word-wrap: break-word;">"success-file"</td>
diff --git a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
index d965fd142..18dae0d38 100644
--- a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
+++ b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
@@ -250,6 +250,14 @@ public class CoreOptions implements Serializable {
"The default partition name in case the dynamic
partition"
+ " column value is null/empty string.");
+ public static final ConfigOption<Boolean> PARTITION_GENERATE_LEGCY_NAME =
+ key("partition.legacy-name")
+ .booleanType()
+ .defaultValue(true)
+ .withDescription(
+ "The legacy partition name is using `toString` fpr
all types. If false, using "
+ + "cast to string for all types.");
+
public static final ConfigOption<Integer> SNAPSHOT_NUM_RETAINED_MIN =
key("snapshot.num-retained.min")
.intType()
@@ -1539,6 +1547,10 @@ public class CoreOptions implements Serializable {
return options.get(PARTITION_DEFAULT_NAME);
}
+ public boolean legacyPartitionName() {
+ return options.get(PARTITION_GENERATE_LEGCY_NAME);
+ }
+
public boolean sortBySize() {
return options.get(SORT_RANG_STRATEGY) == RangeStrategy.SIZE;
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/casting/AbstractCastRule.java
b/paimon-common/src/main/java/org/apache/paimon/casting/AbstractCastRule.java
similarity index 100%
rename from
paimon-core/src/main/java/org/apache/paimon/casting/AbstractCastRule.java
rename to
paimon-common/src/main/java/org/apache/paimon/casting/AbstractCastRule.java
diff --git
a/paimon-core/src/main/java/org/apache/paimon/casting/BinaryToBinaryCastRule.java
b/paimon-common/src/main/java/org/apache/paimon/casting/BinaryToBinaryCastRule.java
similarity index 100%
rename from
paimon-core/src/main/java/org/apache/paimon/casting/BinaryToBinaryCastRule.java
rename to
paimon-common/src/main/java/org/apache/paimon/casting/BinaryToBinaryCastRule.java
diff --git
a/paimon-core/src/main/java/org/apache/paimon/casting/BinaryToStringCastRule.java
b/paimon-common/src/main/java/org/apache/paimon/casting/BinaryToStringCastRule.java
similarity index 100%
rename from
paimon-core/src/main/java/org/apache/paimon/casting/BinaryToStringCastRule.java
rename to
paimon-common/src/main/java/org/apache/paimon/casting/BinaryToStringCastRule.java
diff --git
a/paimon-core/src/main/java/org/apache/paimon/casting/BooleanToNumericCastRule.java
b/paimon-common/src/main/java/org/apache/paimon/casting/BooleanToNumericCastRule.java
similarity index 100%
rename from
paimon-core/src/main/java/org/apache/paimon/casting/BooleanToNumericCastRule.java
rename to
paimon-common/src/main/java/org/apache/paimon/casting/BooleanToNumericCastRule.java
diff --git
a/paimon-core/src/main/java/org/apache/paimon/casting/BooleanToStringCastRule.java
b/paimon-common/src/main/java/org/apache/paimon/casting/BooleanToStringCastRule.java
similarity index 100%
rename from
paimon-core/src/main/java/org/apache/paimon/casting/BooleanToStringCastRule.java
rename to
paimon-common/src/main/java/org/apache/paimon/casting/BooleanToStringCastRule.java
diff --git
a/paimon-core/src/main/java/org/apache/paimon/casting/CastExecutor.java
b/paimon-common/src/main/java/org/apache/paimon/casting/CastExecutor.java
similarity index 100%
rename from
paimon-core/src/main/java/org/apache/paimon/casting/CastExecutor.java
rename to
paimon-common/src/main/java/org/apache/paimon/casting/CastExecutor.java
diff --git
a/paimon-core/src/main/java/org/apache/paimon/casting/CastExecutors.java
b/paimon-common/src/main/java/org/apache/paimon/casting/CastExecutors.java
similarity index 100%
rename from
paimon-core/src/main/java/org/apache/paimon/casting/CastExecutors.java
rename to
paimon-common/src/main/java/org/apache/paimon/casting/CastExecutors.java
diff --git
a/paimon-core/src/main/java/org/apache/paimon/casting/CastFieldGetter.java
b/paimon-common/src/main/java/org/apache/paimon/casting/CastFieldGetter.java
similarity index 100%
rename from
paimon-core/src/main/java/org/apache/paimon/casting/CastFieldGetter.java
rename to
paimon-common/src/main/java/org/apache/paimon/casting/CastFieldGetter.java
diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/CastRule.java
b/paimon-common/src/main/java/org/apache/paimon/casting/CastRule.java
similarity index 100%
rename from paimon-core/src/main/java/org/apache/paimon/casting/CastRule.java
rename to paimon-common/src/main/java/org/apache/paimon/casting/CastRule.java
diff --git
a/paimon-core/src/main/java/org/apache/paimon/casting/CastRulePredicate.java
b/paimon-common/src/main/java/org/apache/paimon/casting/CastRulePredicate.java
similarity index 100%
rename from
paimon-core/src/main/java/org/apache/paimon/casting/CastRulePredicate.java
rename to
paimon-common/src/main/java/org/apache/paimon/casting/CastRulePredicate.java
diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/CastedRow.java
b/paimon-common/src/main/java/org/apache/paimon/casting/CastedRow.java
similarity index 100%
rename from paimon-core/src/main/java/org/apache/paimon/casting/CastedRow.java
rename to paimon-common/src/main/java/org/apache/paimon/casting/CastedRow.java
diff --git
a/paimon-core/src/main/java/org/apache/paimon/casting/DateToStringCastRule.java
b/paimon-common/src/main/java/org/apache/paimon/casting/DateToStringCastRule.java
similarity index 100%
rename from
paimon-core/src/main/java/org/apache/paimon/casting/DateToStringCastRule.java
rename to
paimon-common/src/main/java/org/apache/paimon/casting/DateToStringCastRule.java
diff --git
a/paimon-core/src/main/java/org/apache/paimon/casting/DateToTimestampCastRule.java
b/paimon-common/src/main/java/org/apache/paimon/casting/DateToTimestampCastRule.java
similarity index 100%
rename from
paimon-core/src/main/java/org/apache/paimon/casting/DateToTimestampCastRule.java
rename to
paimon-common/src/main/java/org/apache/paimon/casting/DateToTimestampCastRule.java
diff --git
a/paimon-core/src/main/java/org/apache/paimon/casting/DecimalToDecimalCastRule.java
b/paimon-common/src/main/java/org/apache/paimon/casting/DecimalToDecimalCastRule.java
similarity index 100%
rename from
paimon-core/src/main/java/org/apache/paimon/casting/DecimalToDecimalCastRule.java
rename to
paimon-common/src/main/java/org/apache/paimon/casting/DecimalToDecimalCastRule.java
diff --git
a/paimon-core/src/main/java/org/apache/paimon/casting/DecimalToNumericPrimitiveCastRule.java
b/paimon-common/src/main/java/org/apache/paimon/casting/DecimalToNumericPrimitiveCastRule.java
similarity index 100%
rename from
paimon-core/src/main/java/org/apache/paimon/casting/DecimalToNumericPrimitiveCastRule.java
rename to
paimon-common/src/main/java/org/apache/paimon/casting/DecimalToNumericPrimitiveCastRule.java
diff --git
a/paimon-core/src/main/java/org/apache/paimon/casting/DefaultValueRow.java
b/paimon-common/src/main/java/org/apache/paimon/casting/DefaultValueRow.java
similarity index 100%
rename from
paimon-core/src/main/java/org/apache/paimon/casting/DefaultValueRow.java
rename to
paimon-common/src/main/java/org/apache/paimon/casting/DefaultValueRow.java
diff --git
a/paimon-core/src/main/java/org/apache/paimon/casting/NumericPrimitiveCastRule.java
b/paimon-common/src/main/java/org/apache/paimon/casting/NumericPrimitiveCastRule.java
similarity index 100%
rename from
paimon-core/src/main/java/org/apache/paimon/casting/NumericPrimitiveCastRule.java
rename to
paimon-common/src/main/java/org/apache/paimon/casting/NumericPrimitiveCastRule.java
diff --git
a/paimon-core/src/main/java/org/apache/paimon/casting/NumericPrimitiveToDecimalCastRule.java
b/paimon-common/src/main/java/org/apache/paimon/casting/NumericPrimitiveToDecimalCastRule.java
similarity index 100%
rename from
paimon-core/src/main/java/org/apache/paimon/casting/NumericPrimitiveToDecimalCastRule.java
rename to
paimon-common/src/main/java/org/apache/paimon/casting/NumericPrimitiveToDecimalCastRule.java
diff --git
a/paimon-core/src/main/java/org/apache/paimon/casting/NumericPrimitiveToTimestamp.java
b/paimon-common/src/main/java/org/apache/paimon/casting/NumericPrimitiveToTimestamp.java
similarity index 100%
rename from
paimon-core/src/main/java/org/apache/paimon/casting/NumericPrimitiveToTimestamp.java
rename to
paimon-common/src/main/java/org/apache/paimon/casting/NumericPrimitiveToTimestamp.java
diff --git
a/paimon-core/src/main/java/org/apache/paimon/casting/NumericToBooleanCastRule.java
b/paimon-common/src/main/java/org/apache/paimon/casting/NumericToBooleanCastRule.java
similarity index 100%
rename from
paimon-core/src/main/java/org/apache/paimon/casting/NumericToBooleanCastRule.java
rename to
paimon-common/src/main/java/org/apache/paimon/casting/NumericToBooleanCastRule.java
diff --git
a/paimon-core/src/main/java/org/apache/paimon/casting/NumericToStringCastRule.java
b/paimon-common/src/main/java/org/apache/paimon/casting/NumericToStringCastRule.java
similarity index 100%
rename from
paimon-core/src/main/java/org/apache/paimon/casting/NumericToStringCastRule.java
rename to
paimon-common/src/main/java/org/apache/paimon/casting/NumericToStringCastRule.java
diff --git
a/paimon-core/src/main/java/org/apache/paimon/casting/StringToBinaryCastRule.java
b/paimon-common/src/main/java/org/apache/paimon/casting/StringToBinaryCastRule.java
similarity index 100%
rename from
paimon-core/src/main/java/org/apache/paimon/casting/StringToBinaryCastRule.java
rename to
paimon-common/src/main/java/org/apache/paimon/casting/StringToBinaryCastRule.java
diff --git
a/paimon-core/src/main/java/org/apache/paimon/casting/StringToBooleanCastRule.java
b/paimon-common/src/main/java/org/apache/paimon/casting/StringToBooleanCastRule.java
similarity index 100%
rename from
paimon-core/src/main/java/org/apache/paimon/casting/StringToBooleanCastRule.java
rename to
paimon-common/src/main/java/org/apache/paimon/casting/StringToBooleanCastRule.java
diff --git
a/paimon-core/src/main/java/org/apache/paimon/casting/StringToDateCastRule.java
b/paimon-common/src/main/java/org/apache/paimon/casting/StringToDateCastRule.java
similarity index 100%
rename from
paimon-core/src/main/java/org/apache/paimon/casting/StringToDateCastRule.java
rename to
paimon-common/src/main/java/org/apache/paimon/casting/StringToDateCastRule.java
diff --git
a/paimon-core/src/main/java/org/apache/paimon/casting/StringToDecimalCastRule.java
b/paimon-common/src/main/java/org/apache/paimon/casting/StringToDecimalCastRule.java
similarity index 100%
rename from
paimon-core/src/main/java/org/apache/paimon/casting/StringToDecimalCastRule.java
rename to
paimon-common/src/main/java/org/apache/paimon/casting/StringToDecimalCastRule.java
diff --git
a/paimon-core/src/main/java/org/apache/paimon/casting/StringToNumericPrimitiveCastRule.java
b/paimon-common/src/main/java/org/apache/paimon/casting/StringToNumericPrimitiveCastRule.java
similarity index 100%
rename from
paimon-core/src/main/java/org/apache/paimon/casting/StringToNumericPrimitiveCastRule.java
rename to
paimon-common/src/main/java/org/apache/paimon/casting/StringToNumericPrimitiveCastRule.java
diff --git
a/paimon-core/src/main/java/org/apache/paimon/casting/StringToStringCastRule.java
b/paimon-common/src/main/java/org/apache/paimon/casting/StringToStringCastRule.java
similarity index 100%
rename from
paimon-core/src/main/java/org/apache/paimon/casting/StringToStringCastRule.java
rename to
paimon-common/src/main/java/org/apache/paimon/casting/StringToStringCastRule.java
diff --git
a/paimon-core/src/main/java/org/apache/paimon/casting/StringToTimeCastRule.java
b/paimon-common/src/main/java/org/apache/paimon/casting/StringToTimeCastRule.java
similarity index 100%
rename from
paimon-core/src/main/java/org/apache/paimon/casting/StringToTimeCastRule.java
rename to
paimon-common/src/main/java/org/apache/paimon/casting/StringToTimeCastRule.java
diff --git
a/paimon-core/src/main/java/org/apache/paimon/casting/StringToTimestampCastRule.java
b/paimon-common/src/main/java/org/apache/paimon/casting/StringToTimestampCastRule.java
similarity index 100%
rename from
paimon-core/src/main/java/org/apache/paimon/casting/StringToTimestampCastRule.java
rename to
paimon-common/src/main/java/org/apache/paimon/casting/StringToTimestampCastRule.java
diff --git
a/paimon-core/src/main/java/org/apache/paimon/casting/TimeToStringCastRule.java
b/paimon-common/src/main/java/org/apache/paimon/casting/TimeToStringCastRule.java
similarity index 100%
rename from
paimon-core/src/main/java/org/apache/paimon/casting/TimeToStringCastRule.java
rename to
paimon-common/src/main/java/org/apache/paimon/casting/TimeToStringCastRule.java
diff --git
a/paimon-core/src/main/java/org/apache/paimon/casting/TimeToTimestampCastRule.java
b/paimon-common/src/main/java/org/apache/paimon/casting/TimeToTimestampCastRule.java
similarity index 100%
rename from
paimon-core/src/main/java/org/apache/paimon/casting/TimeToTimestampCastRule.java
rename to
paimon-common/src/main/java/org/apache/paimon/casting/TimeToTimestampCastRule.java
diff --git
a/paimon-core/src/main/java/org/apache/paimon/casting/TimestampToDateCastRule.java
b/paimon-common/src/main/java/org/apache/paimon/casting/TimestampToDateCastRule.java
similarity index 100%
rename from
paimon-core/src/main/java/org/apache/paimon/casting/TimestampToDateCastRule.java
rename to
paimon-common/src/main/java/org/apache/paimon/casting/TimestampToDateCastRule.java
diff --git
a/paimon-core/src/main/java/org/apache/paimon/casting/TimestampToNumericPrimitiveCastRule.java
b/paimon-common/src/main/java/org/apache/paimon/casting/TimestampToNumericPrimitiveCastRule.java
similarity index 100%
rename from
paimon-core/src/main/java/org/apache/paimon/casting/TimestampToNumericPrimitiveCastRule.java
rename to
paimon-common/src/main/java/org/apache/paimon/casting/TimestampToNumericPrimitiveCastRule.java
diff --git
a/paimon-core/src/main/java/org/apache/paimon/casting/TimestampToStringCastRule.java
b/paimon-common/src/main/java/org/apache/paimon/casting/TimestampToStringCastRule.java
similarity index 100%
rename from
paimon-core/src/main/java/org/apache/paimon/casting/TimestampToStringCastRule.java
rename to
paimon-common/src/main/java/org/apache/paimon/casting/TimestampToStringCastRule.java
diff --git
a/paimon-core/src/main/java/org/apache/paimon/casting/TimestampToTimeCastRule.java
b/paimon-common/src/main/java/org/apache/paimon/casting/TimestampToTimeCastRule.java
similarity index 100%
rename from
paimon-core/src/main/java/org/apache/paimon/casting/TimestampToTimeCastRule.java
rename to
paimon-common/src/main/java/org/apache/paimon/casting/TimestampToTimeCastRule.java
diff --git
a/paimon-core/src/main/java/org/apache/paimon/casting/TimestampToTimestampCastRule.java
b/paimon-common/src/main/java/org/apache/paimon/casting/TimestampToTimestampCastRule.java
similarity index 100%
rename from
paimon-core/src/main/java/org/apache/paimon/casting/TimestampToTimestampCastRule.java
rename to
paimon-common/src/main/java/org/apache/paimon/casting/TimestampToTimestampCastRule.java
diff --git
a/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowPartitionComputer.java
b/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowPartitionComputer.java
index 881f0f4d1..1bc8eecc8 100644
---
a/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowPartitionComputer.java
+++
b/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowPartitionComputer.java
@@ -20,6 +20,7 @@ package org.apache.paimon.utils;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.types.DataType;
import org.apache.paimon.types.RowType;
import java.util.Arrays;
@@ -35,11 +36,18 @@ public class InternalRowPartitionComputer {
protected final String defaultPartValue;
protected final String[] partitionColumns;
protected final InternalRow.FieldGetter[] partitionFieldGetters;
+ protected final List<DataType> types;
+ protected final boolean legacyPartitionName;
public InternalRowPartitionComputer(
- String defaultPartValue, RowType rowType, String[]
partitionColumns) {
+ String defaultPartValue,
+ RowType rowType,
+ String[] partitionColumns,
+ boolean legacyPartitionName) {
this.defaultPartValue = defaultPartValue;
this.partitionColumns = partitionColumns;
+ this.types = rowType.getFieldTypes();
+ this.legacyPartitionName = legacyPartitionName;
List<String> columnList = rowType.getFieldNames();
this.partitionFieldGetters =
Arrays.stream(partitionColumns)
@@ -56,7 +64,8 @@ public class InternalRowPartitionComputer {
for (int i = 0; i < partitionFieldGetters.length; i++) {
Object field = partitionFieldGetters[i].getFieldOrNull(in);
- String partitionValue = field != null ? field.toString() : null;
+ String partitionValue =
+ TypeUtils.castPartitionValueToString(field, types.get(i),
legacyPartitionName);
if (StringUtils.isNullOrWhitespaceOnly(partitionValue)) {
partitionValue = defaultPartValue;
}
diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/TypeUtils.java
b/paimon-common/src/main/java/org/apache/paimon/utils/TypeUtils.java
index 8d8d84953..046e3d4b3 100644
--- a/paimon-common/src/main/java/org/apache/paimon/utils/TypeUtils.java
+++ b/paimon-common/src/main/java/org/apache/paimon/utils/TypeUtils.java
@@ -18,6 +18,8 @@
package org.apache.paimon.utils;
+import org.apache.paimon.casting.CastExecutor;
+import org.apache.paimon.casting.CastExecutors;
import org.apache.paimon.data.BinaryString;
import org.apache.paimon.data.Decimal;
import org.apache.paimon.data.GenericArray;
@@ -88,6 +90,24 @@ public class TypeUtils {
.collect(Collectors.toList()));
}
+ public static String castPartitionValueToString(
+ Object value, DataType type, boolean legacyPartitionName) {
+ if (value == null) {
+ return null;
+ }
+ if (legacyPartitionName) {
+ return value.toString();
+ }
+
+ CastExecutor<Object, Object> castExecutor =
+ (CastExecutor<Object, Object>) CastExecutors.resolve(type,
VarCharType.STRING_TYPE);
+ if (castExecutor == null) {
+ throw new UnsupportedOperationException(type + " is not
supported");
+ }
+ Object result = castExecutor.cast(value);
+ return result == null ? null : result.toString();
+ }
+
public static Object castFromString(String s, DataType type) {
return castFromStringInternal(s, type, false);
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/casting/CastExecutorTest.java
b/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java
similarity index 100%
rename from
paimon-core/src/test/java/org/apache/paimon/casting/CastExecutorTest.java
rename to
paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java
diff --git a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java
b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java
index 53bd1291f..811848453 100644
--- a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java
+++ b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java
@@ -105,7 +105,8 @@ abstract class AbstractFileStore<T> implements FileStore<T>
{
options.partitionDefaultName(),
options.fileFormat().getFormatIdentifier(),
options.dataFilePrefix(),
- options.changelogFilePrefix());
+ options.changelogFilePrefix(),
+ options.legacyPartitionName());
}
@Override
diff --git a/paimon-core/src/main/java/org/apache/paimon/KeyValueFileStore.java
b/paimon-core/src/main/java/org/apache/paimon/KeyValueFileStore.java
index cc4579898..2e3aa965f 100644
--- a/paimon-core/src/main/java/org/apache/paimon/KeyValueFileStore.java
+++ b/paimon-core/src/main/java/org/apache/paimon/KeyValueFileStore.java
@@ -206,7 +206,8 @@ public class KeyValueFileStore extends
AbstractFileStore<KeyValue> {
options.partitionDefaultName(),
format,
options.dataFilePrefix(),
- options.changelogFilePrefix())));
+ options.changelogFilePrefix(),
+ options.legacyPartitionName())));
return pathFactoryMap;
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreWrite.java
b/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreWrite.java
index dab20d642..d63887030 100644
---
a/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreWrite.java
+++
b/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreWrite.java
@@ -88,6 +88,7 @@ public abstract class AbstractFileStoreWrite<T> implements
FileStoreWrite<T> {
protected CompactionMetrics compactionMetrics = null;
protected final String tableName;
private boolean isInsertOnly;
+ private boolean legacyPartitionName;
protected AbstractFileStoreWrite(
SnapshotManager snapshotManager,
@@ -97,7 +98,8 @@ public abstract class AbstractFileStoreWrite<T> implements
FileStoreWrite<T> {
String tableName,
int totalBuckets,
RowType partitionType,
- int writerNumberMax) {
+ int writerNumberMax,
+ boolean legacyPartitionName) {
this.snapshotManager = snapshotManager;
this.scan = scan;
this.indexFactory = indexFactory;
@@ -107,6 +109,7 @@ public abstract class AbstractFileStoreWrite<T> implements
FileStoreWrite<T> {
this.writers = new HashMap<>();
this.tableName = tableName;
this.writerNumberMax = writerNumberMax;
+ this.legacyPartitionName = legacyPartitionName;
}
@Override
@@ -469,7 +472,8 @@ public abstract class AbstractFileStoreWrite<T> implements
FileStoreWrite<T> {
? "partition "
+ getPartitionComputer(
partitionType,
-
PARTITION_DEFAULT_NAME.defaultValue())
+
PARTITION_DEFAULT_NAME.defaultValue(),
+ legacyPartitionName)
.generatePartValues(partition)
: "table";
throw new RuntimeException(
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/MemoryFileStoreWrite.java
b/paimon-core/src/main/java/org/apache/paimon/operation/MemoryFileStoreWrite.java
index b7feeead4..bbde3fd48 100644
---
a/paimon-core/src/main/java/org/apache/paimon/operation/MemoryFileStoreWrite.java
+++
b/paimon-core/src/main/java/org/apache/paimon/operation/MemoryFileStoreWrite.java
@@ -75,7 +75,8 @@ public abstract class MemoryFileStoreWrite<T> extends
AbstractFileStoreWrite<T>
tableName,
options.bucket(),
partitionType,
- options.writeMaxWritersToSpill());
+ options.writeMaxWritersToSpill(),
+ options.legacyPartitionName());
this.options = options;
this.cacheManager = new CacheManager(options.lookupCacheMaxMemory());
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/utils/FileStorePathFactory.java
b/paimon-core/src/main/java/org/apache/paimon/utils/FileStorePathFactory.java
index df442383a..612565c72 100644
---
a/paimon-core/src/main/java/org/apache/paimon/utils/FileStorePathFactory.java
+++
b/paimon-core/src/main/java/org/apache/paimon/utils/FileStorePathFactory.java
@@ -56,11 +56,13 @@ public class FileStorePathFactory {
String defaultPartValue,
String formatIdentifier,
String dataFilePrefix,
- String changelogFilePrefix) {
+ String changelogFilePrefix,
+ boolean legacyPartitionName) {
this.root = root;
this.uuid = UUID.randomUUID().toString();
- this.partitionComputer = getPartitionComputer(partitionType,
defaultPartValue);
+ this.partitionComputer =
+ getPartitionComputer(partitionType, defaultPartValue,
legacyPartitionName);
this.formatIdentifier = formatIdentifier;
this.dataFilePrefix = dataFilePrefix;
this.changelogFilePrefix = changelogFilePrefix;
@@ -78,9 +80,10 @@ public class FileStorePathFactory {
@VisibleForTesting
public static InternalRowPartitionComputer getPartitionComputer(
- RowType partitionType, String defaultPartValue) {
+ RowType partitionType, String defaultPartValue, boolean
legacyPartitionName) {
String[] partitionColumns = partitionType.getFieldNames().toArray(new
String[0]);
- return new InternalRowPartitionComputer(defaultPartValue,
partitionType, partitionColumns);
+ return new InternalRowPartitionComputer(
+ defaultPartValue, partitionType, partitionColumns,
legacyPartitionName);
}
public Path newManifestFile() {
diff --git
a/paimon-core/src/test/java/org/apache/paimon/io/KeyValueFileReadWriteTest.java
b/paimon-core/src/test/java/org/apache/paimon/io/KeyValueFileReadWriteTest.java
index 3c8955231..de593e80b 100644
---
a/paimon-core/src/test/java/org/apache/paimon/io/KeyValueFileReadWriteTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/io/KeyValueFileReadWriteTest.java
@@ -228,7 +228,8 @@ public class KeyValueFileReadWriteTest {
CoreOptions.PARTITION_DEFAULT_NAME.defaultValue(),
format,
CoreOptions.DATA_FILE_PREFIX.defaultValue(),
- CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue());
+ CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(),
+
CoreOptions.PARTITION_GENERATE_LEGCY_NAME.defaultValue());
int suggestedFileSize = ThreadLocalRandom.current().nextInt(8192) +
1024;
FileIO fileIO = FileIOFinder.find(path);
Options options = new Options();
@@ -244,7 +245,8 @@ public class KeyValueFileReadWriteTest {
CoreOptions.PARTITION_DEFAULT_NAME.defaultValue(),
CoreOptions.FILE_FORMAT.defaultValue().toString(),
CoreOptions.DATA_FILE_PREFIX.defaultValue(),
- CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue()));
+ CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(),
+
CoreOptions.PARTITION_GENERATE_LEGCY_NAME.defaultValue()));
return KeyValueFileWriterFactory.builder(
fileIO,
diff --git
a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTestBase.java
b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTestBase.java
index 31be9346c..87a6f2e45 100644
---
a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTestBase.java
+++
b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTestBase.java
@@ -146,7 +146,8 @@ public abstract class ManifestFileMetaTestBase {
"default",
CoreOptions.FILE_FORMAT.defaultValue(),
CoreOptions.DATA_FILE_PREFIX.defaultValue(),
-
CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue()),
+
CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(),
+
CoreOptions.PARTITION_GENERATE_LEGCY_NAME.defaultValue()),
Long.MAX_VALUE,
null)
.create();
diff --git
a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java
b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java
index a11f3e485..30e8e7b20 100644
--- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java
@@ -102,7 +102,8 @@ public class ManifestFileTest {
"default",
CoreOptions.FILE_FORMAT.defaultValue().toString(),
CoreOptions.DATA_FILE_PREFIX.defaultValue(),
- CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue());
+ CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(),
+
CoreOptions.PARTITION_GENERATE_LEGCY_NAME.defaultValue());
int suggestedFileSize = ThreadLocalRandom.current().nextInt(8192) +
1024;
FileIO fileIO = FileIOFinder.find(path);
return new ManifestFile.Factory(
diff --git
a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestListTest.java
b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestListTest.java
index faf1004e1..f4703c1ff 100644
--- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestListTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestListTest.java
@@ -106,7 +106,8 @@ public class ManifestListTest {
"default",
CoreOptions.FILE_FORMAT.defaultValue().toString(),
CoreOptions.DATA_FILE_PREFIX.defaultValue(),
- CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue());
+ CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(),
+
CoreOptions.PARTITION_GENERATE_LEGCY_NAME.defaultValue());
return new ManifestList.Factory(FileIOFinder.find(path), avro, "zstd",
pathFactory, null)
.create();
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/utils/FileStorePathFactoryTest.java
b/paimon-core/src/test/java/org/apache/paimon/utils/FileStorePathFactoryTest.java
index 994367d0c..26bbd28e4 100644
---
a/paimon-core/src/test/java/org/apache/paimon/utils/FileStorePathFactoryTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/utils/FileStorePathFactoryTest.java
@@ -87,7 +87,8 @@ public class FileStorePathFactoryTest {
"default",
CoreOptions.FILE_FORMAT.defaultValue().toString(),
CoreOptions.DATA_FILE_PREFIX.defaultValue(),
- CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue());
+ CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(),
+
CoreOptions.PARTITION_GENERATE_LEGCY_NAME.defaultValue());
assertPartition("20211224", 16, pathFactory, "/dt=20211224/hr=16");
assertPartition("20211224", null, pathFactory,
"/dt=20211224/hr=default");
@@ -124,6 +125,7 @@ public class FileStorePathFactoryTest {
PARTITION_DEFAULT_NAME.defaultValue(),
CoreOptions.FILE_FORMAT.defaultValue().toString(),
CoreOptions.DATA_FILE_PREFIX.defaultValue(),
- CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue());
+ CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(),
+ CoreOptions.PARTITION_GENERATE_LEGCY_NAME.defaultValue());
}
}
diff --git
a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkCatalog.java
b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkCatalog.java
index 633fa9c4b..c1e7db6b2 100644
---
a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkCatalog.java
+++
b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkCatalog.java
@@ -1075,9 +1075,12 @@ public class FlinkCatalog extends AbstractCatalog {
getPartitionEntries(table, tablePath, partitionSpec);
org.apache.paimon.types.RowType partitionRowType =
table.schema().logicalPartitionType();
+ CoreOptions options = new CoreOptions(table.options());
InternalRowPartitionComputer partitionComputer =
FileStorePathFactory.getPartitionComputer(
- partitionRowType, new
CoreOptions(table.options()).partitionDefaultName());
+ partitionRowType,
+ options.partitionDefaultName(),
+ options.legacyPartitionName());
return partitionEntries.stream()
.map(
diff --git
a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/partition/PartitionMarkDone.java
b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/partition/PartitionMarkDone.java
index 2c5d55f20..d0825bcdb 100644
---
a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/partition/PartitionMarkDone.java
+++
b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/partition/PartitionMarkDone.java
@@ -71,7 +71,8 @@ public class PartitionMarkDone implements Closeable {
new InternalRowPartitionComputer(
coreOptions.partitionDefaultName(),
table.schema().logicalPartitionType(),
- table.partitionKeys().toArray(new String[0]));
+ table.partitionKeys().toArray(new String[0]),
+ coreOptions.legacyPartitionName());
PartitionMarkDoneTrigger trigger =
PartitionMarkDoneTrigger.create(coreOptions, isRestored,
stateStore);
diff --git
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/TestChangelogDataReadWrite.java
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/TestChangelogDataReadWrite.java
index aab8666fe..762a14ea1 100644
---
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/TestChangelogDataReadWrite.java
+++
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/TestChangelogDataReadWrite.java
@@ -107,7 +107,8 @@ public class TestChangelogDataReadWrite {
"default",
CoreOptions.FILE_FORMAT.defaultValue().toString(),
CoreOptions.DATA_FILE_PREFIX.defaultValue(),
- CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue());
+ CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(),
+
CoreOptions.PARTITION_GENERATE_LEGCY_NAME.defaultValue());
this.snapshotManager = new SnapshotManager(LocalFileIO.create(), new
Path(root));
this.commitUser = UUID.randomUUID().toString();
}
diff --git
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveMetastoreClient.java
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveMetastoreClient.java
index b006f09c9..6ec3b086d 100644
---
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveMetastoreClient.java
+++
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveMetastoreClient.java
@@ -56,11 +56,13 @@ public class HiveMetastoreClient implements MetastoreClient
{
ClientPool<IMetaStoreClient, TException> clients)
throws TException, InterruptedException {
this.identifier = identifier;
+ CoreOptions options = new CoreOptions(schema.options());
this.partitionComputer =
new InternalRowPartitionComputer(
- new
CoreOptions(schema.options()).partitionDefaultName(),
+ options.partitionDefaultName(),
schema.logicalPartitionType(),
- schema.partitionKeys().toArray(new String[0]));
+ schema.partitionKeys().toArray(new String[0]),
+ options.legacyPartitionName());
this.clients = clients;
this.sd =
diff --git
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala
index cbdbd67a6..54970bfe3 100644
---
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala
+++
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala
@@ -18,6 +18,7 @@
package org.apache.paimon.spark
+import org.apache.paimon.CoreOptions
import org.apache.paimon.metastore.MetastoreClient
import org.apache.paimon.operation.FileStoreCommit
import org.apache.paimon.table.FileStoreTable
@@ -31,7 +32,6 @@ import org.apache.spark.sql.catalyst.util.CharVarcharUtils
import org.apache.spark.sql.connector.catalog.SupportsAtomicPartitionManagement
import org.apache.spark.sql.types.StructType
-import java.util
import java.util.{Map => JMap, Objects, UUID}
import scala.collection.JavaConverters._
@@ -52,7 +52,8 @@ trait PaimonPartitionManagement extends
SupportsAtomicPartitionManagement {
val rowDataPartitionComputer = new InternalRowPartitionComputer(
fileStoreTable.coreOptions().partitionDefaultName(),
partitionRowType,
- table.partitionKeys().asScala.toArray)
+ table.partitionKeys().asScala.toArray,
+ CoreOptions.fromMap(table.options()).legacyPartitionName)
rows.map {
r =>
diff --git
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/DeleteFromPaimonTableCommand.scala
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/DeleteFromPaimonTableCommand.scala
index 0087ffb74..bff69aa6f 100644
---
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/DeleteFromPaimonTableCommand.scala
+++
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/DeleteFromPaimonTableCommand.scala
@@ -83,7 +83,8 @@ case class DeleteFromPaimonTableCommand(
val rowDataPartitionComputer = new InternalRowPartitionComputer(
table.coreOptions().partitionDefaultName(),
table.schema().logicalPartitionType(),
- table.partitionKeys.asScala.toArray
+ table.partitionKeys.asScala.toArray,
+ table.coreOptions().legacyPartitionName()
)
val dropPartitions = matchedPartitions.map {
partition =>
rowDataPartitionComputer.generatePartValues(partition).asScala.asJava
diff --git
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/sources/StreamHelper.scala
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/sources/StreamHelper.scala
index 91df04e6d..b44a66fce 100644
---
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/sources/StreamHelper.scala
+++
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/sources/StreamHelper.scala
@@ -49,12 +49,15 @@ private[spark] trait StreamHelper {
private lazy val partitionSchema: StructType =
SparkTypeUtils.fromPaimonRowType(TypeUtils.project(table.rowType(),
table.partitionKeys()))
- private lazy val partitionComputer: InternalRowPartitionComputer =
+ private lazy val partitionComputer: InternalRowPartitionComputer = {
+ val options = new CoreOptions(table.options)
new InternalRowPartitionComputer(
- new CoreOptions(table.options).partitionDefaultName,
+ options.partitionDefaultName,
TypeUtils.project(table.rowType(), table.partitionKeys()),
- table.partitionKeys().asScala.toArray
+ table.partitionKeys().asScala.toArray,
+ options.legacyPartitionName()
)
+ }
// Used to get the initial offset.
lazy val streamScanStartingContext: StartingContext =
streamScan.startingContext()
diff --git
a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkFileIndexITCase.java
b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkFileIndexITCase.java
index 7af83186e..d82f92af5 100644
---
a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkFileIndexITCase.java
+++
b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkFileIndexITCase.java
@@ -130,7 +130,8 @@ public class SparkFileIndexITCase extends SparkWriteITCase {
new CoreOptions(new Options()).partitionDefaultName(),
CoreOptions.FILE_FORMAT.defaultValue().toString(),
CoreOptions.DATA_FILE_PREFIX.defaultValue(),
- CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue());
+ CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(),
+
CoreOptions.PARTITION_GENERATE_LEGCY_NAME.defaultValue());
Table table = fileSystemCatalog.getTable(Identifier.create("db", "T"));
ReadBuilder readBuilder = table.newReadBuilder();
diff --git
a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DDLTestBase.scala
b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DDLTestBase.scala
index f1215ede4..0006e45ec 100644
---
a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DDLTestBase.scala
+++
b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DDLTestBase.scala
@@ -27,7 +27,7 @@ import org.apache.spark.SparkException
import org.apache.spark.sql.Row
import org.junit.jupiter.api.Assertions
-import java.sql.Timestamp
+import java.sql.{Date, Timestamp}
import java.time.LocalDateTime
abstract class DDLTestBase extends PaimonSparkTestBase {
@@ -495,4 +495,55 @@ abstract class DDLTestBase extends PaimonSparkTestBase {
}.getMessage
assert(error.contains("Unsupported partition transform"))
}
+
+ test("Fix partition column generate wrong partition spec") {
+ Seq(true, false).foreach {
+ legacyPartName =>
+ withTable("p_t") {
+ spark.sql(s"""
+ |CREATE TABLE p_t (
+ | id BIGINT,
+ | c1 STRING
+ |) using paimon
+ |PARTITIONED BY (day binary)
+
|tblproperties('partition.legacy-name'='$legacyPartName');
+ |""".stripMargin)
+
+ if (legacyPartName) {
+ spark.sql("insert into table p_t values(1, 'a', cast('2021' as
binary))")
+ intercept[Exception] {
+ spark.sql("SELECT * FROM p_t").collect()
+ }
+ } else {
+ spark.sql("insert into table p_t values(1, 'a', cast('2021' as
binary))")
+ checkAnswer(spark.sql("SELECT * FROM p_t"), Row(1, "a",
"2021".getBytes))
+ val path = spark.sql("SELECT __paimon_file_path FROM
p_t").collect()
+ assert(path.length == 1)
+ assert(path.head.getString(0).contains("/day=2021/"))
+ }
+ }
+
+ withTable("p_t") {
+ spark.sql(s"""
+ |CREATE TABLE p_t (
+ | id BIGINT,
+ | c1 STRING
+ |) using paimon
+ |PARTITIONED BY (day date)
+
|tblproperties('partition.legacy-name'='$legacyPartName');
+ |""".stripMargin)
+
+ spark.sql("insert into table p_t values(1, 'a', cast('2021-01-01' as
date))")
+ checkAnswer(spark.sql("SELECT * FROM p_t"), Row(1, "a",
Date.valueOf("2021-01-01")))
+
+ val path = spark.sql("SELECT __paimon_file_path FROM p_t").collect()
+ assert(path.length == 1)
+ if (legacyPartName) {
+ assert(path.head.getString(0).contains("/day=18628/"))
+ } else {
+ assert(path.head.getString(0).contains("/day=2021-01-01/"))
+ }
+ }
+ }
+ }
}
