This is an automated email from the ASF dual-hosted git repository.
xushiyan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 80783c27f5 [HUDI-4187] Fix partition order in aws glue sync (#5731)
80783c27f5 is described below
commit 80783c27f5211ae5cd01d692223a106df5dff4a2
Author: Nicolas Paris <[email protected]>
AuthorDate: Sat Jun 4 11:16:52 2022 +0200
[HUDI-4187] Fix partition order in aws glue sync (#5731)
---
.../org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git
a/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java
b/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java
index e5a23a9a57..f970423b33 100644
---
a/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java
+++
b/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java
@@ -271,19 +271,22 @@ public class AWSGlueCatalogSyncClient extends
AbstractHiveSyncHoodieClient {
try {
Map<String, String> mapSchema = parquetSchemaToMapSchema(storageSchema,
syncConfig.supportTimestamp, false);
- List<Column> schemaPartitionKeys = new ArrayList<>();
List<Column> schemaWithoutPartitionKeys = new ArrayList<>();
for (String key : mapSchema.keySet()) {
String keyType = getPartitionKeyType(mapSchema, key);
Column column = new
Column().withName(key).withType(keyType.toLowerCase()).withComment("");
// In Glue, the full schema should exclude the partition keys
- if (syncConfig.partitionFields.contains(key)) {
- schemaPartitionKeys.add(column);
- } else {
+ if (!syncConfig.partitionFields.contains(key)) {
schemaWithoutPartitionKeys.add(column);
}
}
+ // now create the schema partition
+ List<Column> schemaPartitionKeys =
syncConfig.partitionFields.stream().map(partitionKey -> {
+ String keyType = getPartitionKeyType(mapSchema, partitionKey);
+ return new
Column().withName(partitionKey).withType(keyType.toLowerCase()).withComment("");
+ }).collect(Collectors.toList());
+
StorageDescriptor storageDescriptor = new StorageDescriptor();
serdeProperties.put("serialization.format", "1");
storageDescriptor
