This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 667626cb9b37 chore(glue-sync): Ignore EntityNotFoundException when
dropping Glue partitions (#19142)
667626cb9b37 is described below
commit 667626cb9b37719d222b7840cdb5ab8f0a41110d
Author: wangxianghu <[email protected]>
AuthorDate: Fri Jul 3 09:51:35 2026 +0800
chore(glue-sync): Ignore EntityNotFoundException when dropping Glue
partitions (#19142)
---
.../hudi/aws/sync/AWSGlueCatalogSyncClient.java | 20 +++++++-
.../hudi/aws/sync/TestAWSGlueSyncClient.java | 56 ++++++++++++++++++++++
2 files changed, 74 insertions(+), 2 deletions(-)
diff --git
a/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java
b/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java
index 884b5c53a25d..10f0b5a1592d 100644
---
a/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java
+++
b/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java
@@ -71,6 +71,7 @@ import
software.amazon.awssdk.services.glue.model.GetPartitionsRequest;
import software.amazon.awssdk.services.glue.model.GetPartitionsResponse;
import software.amazon.awssdk.services.glue.model.GetTableRequest;
import software.amazon.awssdk.services.glue.model.KeySchemaElement;
+import software.amazon.awssdk.services.glue.model.PartitionError;
import software.amazon.awssdk.services.glue.model.PartitionIndex;
import software.amazon.awssdk.services.glue.model.PartitionIndexDescriptor;
import software.amazon.awssdk.services.glue.model.PartitionInput;
@@ -137,6 +138,7 @@ public class AWSGlueCatalogSyncClient extends
HoodieSyncClient {
private static final int MAX_PARTITIONS_PER_CHANGE_REQUEST = 100;
private static final int MAX_PARTITIONS_PER_READ_REQUEST = 1000;
private static final int MAX_DELETE_PARTITIONS_PER_REQUEST = 25;
+ private static final String ENTITY_NOT_FOUND_ERROR_CODE =
"EntityNotFoundException";
protected final GlueAsyncClient awsGlue;
private static final String GLUE_PARTITION_INDEX_ENABLE =
"partition_filtering.enabled";
private static final int PARTITION_INDEX_MAX_NUMBER = 3;
@@ -437,8 +439,22 @@ public class AWSGlueCatalogSyncClient extends
HoodieSyncClient {
BatchDeletePartitionResponse response = future.get();
if (CollectionUtils.nonEmpty(response.errors())) {
- throw new HoodieGlueSyncException("Fail to drop partitions to " +
tableId(databaseName, tableName)
- + " with error(s): " + response.errors());
+ // Dropping a partition that no longer exists is a no-op for an
idempotent cleanup, so
+ // ignore EntityNotFoundException errors and only fail on other (e.g.
permission/throttling) errors.
+ Map<Boolean, List<PartitionError>> errorsByIgnorable =
response.errors().stream()
+ .collect(Collectors.partitioningBy(
+ error ->
ENTITY_NOT_FOUND_ERROR_CODE.equals(error.errorDetail().errorCode())));
+ List<PartitionError> ignorableErrors = errorsByIgnorable.get(true);
+ if (!ignorableErrors.isEmpty()) {
+ log.info("Ignored dropping {} non-existent partition(s) from table
{}: {}", ignorableErrors.size(),
+ tableId(databaseName, tableName),
+
ignorableErrors.stream().map(PartitionError::partitionValues).collect(Collectors.toList()));
+ }
+ List<PartitionError> realErrors = errorsByIgnorable.get(false);
+ if (!realErrors.isEmpty()) {
+ throw new HoodieGlueSyncException("Fail to drop partitions to " +
tableId(databaseName, tableName)
+ + " with error(s): " + realErrors);
+ }
}
} catch (Exception e) {
throw new HoodieGlueSyncException("Fail to drop partitions to " +
tableId(databaseName, tableName), e);
diff --git
a/hudi-aws/src/test/java/org/apache/hudi/aws/sync/TestAWSGlueSyncClient.java
b/hudi-aws/src/test/java/org/apache/hudi/aws/sync/TestAWSGlueSyncClient.java
index f4822e32f05d..8c3b24c0b677 100644
--- a/hudi-aws/src/test/java/org/apache/hudi/aws/sync/TestAWSGlueSyncClient.java
+++ b/hudi-aws/src/test/java/org/apache/hudi/aws/sync/TestAWSGlueSyncClient.java
@@ -647,6 +647,62 @@ class TestAWSGlueSyncClient {
assertTrue(ex.getCause().getCause().getMessage().contains("Fail to drop
partitions"));
}
+ @Test
+ void testDropPartitions_IgnoresEntityNotFound() {
+ String tableName = "tbl";
+ List<String> toDrop = List.of("2025/05/19");
+
+ // Glue reports EntityNotFoundException for a partition that no longer
exists; it should be ignored.
+ ErrorDetail detail =
ErrorDetail.builder().errorCode(EntityNotFoundException.class.getSimpleName()).build();
+ PartitionError pe =
PartitionError.builder().partitionValues(Arrays.asList("2025", "05",
"19")).errorDetail(detail).build();
+ BatchDeletePartitionResponse resp = BatchDeletePartitionResponse.builder()
+ .errors(Collections.singletonList(pe))
+ .build();
+
when(mockAwsGlue.batchDeletePartition(any(BatchDeletePartitionRequest.class)))
+ .thenReturn(CompletableFuture.completedFuture(resp));
+
+ // should swallow the EntityNotFound error and not throw
+ awsGlueSyncClient.dropPartitions(tableName, toDrop);
+
+
verify(mockAwsGlue).batchDeletePartition(any(BatchDeletePartitionRequest.class));
+ }
+
+ @Test
+ void testDropPartitions_MixedErrorsStillThrow() {
+ String tableName = "tbl";
+ List<String> toDrop = Arrays.asList("2025/05/19", "2025/05/18");
+
+ // One ignorable EntityNotFound error and one real error -> should still
throw for the real one.
+ PartitionError ignorable = PartitionError.builder()
+ .partitionValues(Arrays.asList("2025", "05", "19"))
+
.errorDetail(ErrorDetail.builder().errorCode(EntityNotFoundException.class.getSimpleName()).build())
+ .build();
+ PartitionError real = PartitionError.builder()
+ .partitionValues(Arrays.asList("2025", "05", "18"))
+
.errorDetail(ErrorDetail.builder().errorCode("InternalServiceException").build())
+ .build();
+ BatchDeletePartitionResponse resp = BatchDeletePartitionResponse.builder()
+ .errors(Arrays.asList(ignorable, real))
+ .build();
+
when(mockAwsGlue.batchDeletePartition(any(BatchDeletePartitionRequest.class)))
+ .thenReturn(CompletableFuture.completedFuture(resp));
+
+ HoodieGlueSyncException ex = assertThrows(
+ HoodieGlueSyncException.class,
+ () -> awsGlueSyncClient.dropPartitions(tableName, toDrop)
+ );
+ // Walk the full cause chain: the error list is nested a few wrappers deep.
+ StringBuilder chain = new StringBuilder();
+ for (Throwable t = ex; t != null; t = t.getCause()) {
+ chain.append(t.getMessage()).append('\n');
+ }
+ String messages = chain.toString();
+ assertTrue(messages.contains("Fail to drop partitions"));
+ // Only the real error should be surfaced, not the ignored EntityNotFound
one.
+ assertTrue(messages.contains("InternalServiceException"));
+
assertFalse(messages.contains(EntityNotFoundException.class.getSimpleName()));
+ }
+
@Disabled("Integration test – requires real AWS environment")
@Test
void testIntegrationTableExists_RealGlueEnvironment() {
