This is an automated email from the ASF dual-hosted git repository.
taowang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/amoro.git
The following commit(s) were added to refs/heads/master by this push:
new c40685bbe [AMORO-3718]Inconsistency in capitalization between iceberg
and hive … (#3719)
c40685bbe is described below
commit c40685bbe21d18ff05a003faff92b624609e5645
Author: Nico CHen <[email protected]>
AuthorDate: Thu Aug 7 17:06:10 2025 +0800
[AMORO-3718]Inconsistency in capitalization between iceberg and hive …
(#3719)
[AMORO-3718]Inconsistency in capitalization between iceberg and hive
partition name within a mixed hive table
---
.../amoro/hive/utils/HiveMetaSynchronizer.java | 36 +++++++++++++++++++++-
1 file changed, 35 insertions(+), 1 deletion(-)
diff --git
a/amoro-format-mixed/amoro-mixed-hive/src/main/java/org/apache/amoro/hive/utils/HiveMetaSynchronizer.java
b/amoro-format-mixed/amoro-mixed-hive/src/main/java/org/apache/amoro/hive/utils/HiveMetaSynchronizer.java
index 8f1af735d..1bcbbe4e9 100644
---
a/amoro-format-mixed/amoro-mixed-hive/src/main/java/org/apache/amoro/hive/utils/HiveMetaSynchronizer.java
+++
b/amoro-format-mixed/amoro-mixed-hive/src/main/java/org/apache/amoro/hive/utils/HiveMetaSynchronizer.java
@@ -40,6 +40,7 @@ import org.apache.iceberg.DataFile;
import org.apache.iceberg.FileScanTask;
import org.apache.iceberg.MetricsConfig;
import org.apache.iceberg.OverwriteFiles;
+import org.apache.iceberg.PartitionField;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.Schema;
import org.apache.iceberg.StructLike;
@@ -48,6 +49,7 @@ import org.apache.iceberg.UpdateSchema;
import org.apache.iceberg.data.TableMigrationUtil;
import org.apache.iceberg.io.CloseableIterable;
import org.apache.iceberg.mapping.NameMappingParser;
+import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.TypeUtil;
import org.apache.iceberg.types.Types;
import org.apache.iceberg.util.StructLikeMap;
@@ -57,6 +59,8 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.UncheckedIOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@@ -345,11 +349,41 @@ public class HiveMetaSynchronizer {
}
}
+ private static <T> T get(StructLike data, int pos, Class<?> javaClass) {
+ return data.get(pos, (Class<T>) javaClass);
+ }
+
+ private static String escape(String string) {
+ try {
+ return URLEncoder.encode(string, "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public static String partitionToPath(SupportHive table, StructLike data) {
+ StringBuilder sb = new StringBuilder();
+ Class<?>[] javaClasses = table.spec().javaClasses();
+ List<Types.NestedField> outputFields =
table.spec().partitionType().fields();
+ List<PartitionField> fields = table.spec().fields();
+ for (int i = 0; i < javaClasses.length; i += 1) {
+ PartitionField field = fields.get(i);
+ Type type = outputFields.get(i).type();
+ String valueString = field.transform().toHumanString(type, get(data, i,
javaClasses[i]));
+
+ if (i > 0) {
+ sb.append("/");
+ }
+
sb.append(field.name().toLowerCase()).append("=").append(escape(valueString));
+ }
+ return sb.toString();
+ }
+
private static void syncPartitionTable(
SupportHive table, StructLikeMap<Map<String, String>> partitionProperty)
throws Exception {
Map<String, StructLike> icebergPartitionMap = new HashMap<>();
for (StructLike structLike : partitionProperty.keySet()) {
- icebergPartitionMap.put(table.spec().partitionToPath(structLike),
structLike);
+ icebergPartitionMap.put(partitionToPath(table, structLike), structLike);
}
List<String> icebergPartitions = new
ArrayList<>(icebergPartitionMap.keySet());
List<Partition> hivePartitions =
