danny0405 commented on code in PR #10460:
URL: https://github.com/apache/hudi/pull/10460#discussion_r1525655019
##########
hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java:
##########
@@ -249,57 +367,51 @@ public void updatePartitionsToTable(String tableName,
List<String> changedPartit
return
BatchUpdatePartitionRequestEntry.builder().partitionInput(partitionInput).partitionValueList(partitionValues).build();
}).collect(Collectors.toList());
- List<CompletableFuture<BatchUpdatePartitionResponse>> futures = new
ArrayList<>();
- for (List<BatchUpdatePartitionRequestEntry> batch :
CollectionUtils.batches(updatePartitionEntries, MAX_PARTITIONS_PER_REQUEST)) {
- BatchUpdatePartitionRequest request =
BatchUpdatePartitionRequest.builder()
-
.databaseName(databaseName).tableName(tableName).entries(batch).build();
- futures.add(awsGlue.batchUpdatePartition(request));
- }
+ BatchUpdatePartitionRequest request =
BatchUpdatePartitionRequest.builder()
+
.databaseName(databaseName).tableName(table.name()).entries(updatePartitionEntries).build();
+ CompletableFuture<BatchUpdatePartitionResponse> future =
awsGlue.batchUpdatePartition(request);
- for (CompletableFuture<BatchUpdatePartitionResponse> future : futures) {
- BatchUpdatePartitionResponse response = future.get();
- if (CollectionUtils.nonEmpty(response.errors())) {
- throw new HoodieGlueSyncException("Fail to update partitions to " +
tableId(databaseName, tableName)
- + " with error(s): " + response.errors());
- }
+ BatchUpdatePartitionResponse response = future.get();
+ if (CollectionUtils.nonEmpty(response.errors())) {
+ throw new HoodieGlueSyncException("Fail to update partitions to " +
tableId(databaseName, table.name())
+ + " with error(s): " + response.errors());
}
} catch (Exception e) {
- throw new HoodieGlueSyncException("Fail to update partitions to " +
tableId(databaseName, tableName), e);
+ throw new HoodieGlueSyncException("Fail to update partitions to " +
tableId(databaseName, table.name()), e);
}
}
@Override
public void dropPartitions(String tableName, List<String> partitionsToDrop) {
- if (CollectionUtils.isNullOrEmpty(partitionsToDrop)) {
- LOG.info("No partitions to drop for " + tableName);
+ if (partitionsToDrop.isEmpty()) {
+ LOG.info("No partitions to drop for " + tableId(this.databaseName,
tableName));
return;
}
- LOG.info("Drop " + partitionsToDrop.size() + "partition(s) in table " +
tableId(databaseName, tableName));
- try {
- List<CompletableFuture<BatchDeletePartitionResponse>> futures = new
ArrayList<>();
- for (List<String> batch : CollectionUtils.batches(partitionsToDrop,
MAX_DELETE_PARTITIONS_PER_REQUEST)) {
-
- List<PartitionValueList> partitionValueLists =
batch.stream().map(partition -> {
- PartitionValueList partitionValueList = PartitionValueList.builder()
-
.values(partitionValueExtractor.extractPartitionValuesInPath(partition))
- .build();
- return partitionValueList;
- }).collect(Collectors.toList());
+ HoodieTimer timer = HoodieTimer.start();
+ parallelizeChange(partitionsToDrop, this.changeParallelism, partitions ->
this.dropPartitionsInternal(tableName, partitions),
MAX_DELETE_PARTITIONS_PER_REQUEST);
+ LOG.info("Deleted {} partitions to table {} in {} ms",
partitionsToDrop.size(), tableId(this.databaseName, tableName),
timer.endTimer());
+ }
- BatchDeletePartitionRequest batchDeletePartitionRequest =
BatchDeletePartitionRequest.builder()
- .databaseName(databaseName)
- .tableName(tableName)
- .partitionsToDelete(partitionValueLists)
+ private void dropPartitionsInternal(String tableName, List<String>
partitionsToDrop) {
+ try {
+ List<PartitionValueList> partitionValueLists =
partitionsToDrop.stream().map(partition -> {
+ PartitionValueList partitionValueList = PartitionValueList.builder()
Review Comment:
`partitionValueList` can be inlined.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
