Ethan Guo created HUDI-4938:
-------------------------------
Summary: Clean action fails due to IllegalStateException:
Duplicate key
Key: HUDI-4938
URL: https://issues.apache.org/jira/browse/HUDI-4938
Project: Apache Hudi
Issue Type: Bug
Reporter: Ethan Guo
Async clean service alongside Deltastreamer fails due to IllegalStateException
{code:java}
22/09/27 23:15:45 INFO InProcessLockProvider: Thread pool-27-thread-1 RELEASING
in-process lock.
22/09/27 23:15:45 INFO InProcessLockProvider: Thread pool-27-thread-1 RELEASED
in-process lock.
22/09/27 23:15:45 INFO TransactionManager: Transaction ended with transaction
owner Option{val=[==>20220927231527453__clean__INFLIGHT]}
22/09/27 23:15:45 WARN CleanActionExecutor: Failed to perform previous clean
operation, instant: [==>20220927231527453__clean__REQUESTED]
java.lang.IllegalStateException: Duplicate key {"size": 0, "isDeleted": true}
at java.util.stream.Collectors.lambda$throwingMerger$0(Collectors.java:133)
at java.util.HashMap.merge(HashMap.java:1254)
at java.util.stream.Collectors.lambda$toMap$58(Collectors.java:1320)
at java.util.stream.ReduceOps$3ReducingSink.accept(ReduceOps.java:169)
at
java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1384)
at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:482)
at
java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
at
java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:708)
at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
at java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:566)
at
org.apache.hudi.metadata.HoodieMetadataPayload.lambda$createPartitionFilesRecord$7(HoodieMetadataPayload.java:336)
at org.apache.hudi.common.util.Option.ifPresent(Option.java:97)
at
org.apache.hudi.metadata.HoodieMetadataPayload.createPartitionFilesRecord(HoodieMetadataPayload.java:334)
at
org.apache.hudi.metadata.HoodieTableMetadataUtil.lambda$convertMetadataToFilesPartitionRecords$8(HoodieTableMetadataUtil.java:488)
at java.util.HashMap.forEach(HashMap.java:1289)
at
org.apache.hudi.metadata.HoodieTableMetadataUtil.convertMetadataToFilesPartitionRecords(HoodieTableMetadataUtil.java:484)
at
org.apache.hudi.metadata.HoodieTableMetadataUtil.convertMetadataToRecords(HoodieTableMetadataUtil.java:454)
at
org.apache.hudi.metadata.HoodieBackedTableMetadataWriter.lambda$update$13(HoodieBackedTableMetadataWriter.java:892)
at
org.apache.hudi.metadata.HoodieBackedTableMetadataWriter.processAndCommit(HoodieBackedTableMetadataWriter.java:811)
at
org.apache.hudi.metadata.HoodieBackedTableMetadataWriter.update(HoodieBackedTableMetadataWriter.java:892)
at
org.apache.hudi.table.action.BaseActionExecutor.lambda$writeTableMetadata$1(BaseActionExecutor.java:69)
at org.apache.hudi.common.util.Option.ifPresent(Option.java:97)
at
org.apache.hudi.table.action.BaseActionExecutor.writeTableMetadata(BaseActionExecutor.java:69)
at
org.apache.hudi.table.action.clean.CleanActionExecutor.runClean(CleanActionExecutor.java:224)
at
org.apache.hudi.table.action.clean.CleanActionExecutor.runPendingClean(CleanActionExecutor.java:189)
at
org.apache.hudi.table.action.clean.CleanActionExecutor.lambda$execute$8(CleanActionExecutor.java:259)
at java.util.ArrayList.forEach(ArrayList.java:1259)
at
org.apache.hudi.table.action.clean.CleanActionExecutor.execute(CleanActionExecutor.java:253)
at
org.apache.hudi.table.HoodieSparkCopyOnWriteTable.clean(HoodieSparkCopyOnWriteTable.java:269)
at
org.apache.hudi.client.BaseHoodieWriteClient.clean(BaseHoodieWriteClient.java:871)
at
org.apache.hudi.client.BaseHoodieWriteClient.clean(BaseHoodieWriteClient.java:827)
at
org.apache.hudi.async.AsyncCleanerService.lambda$startService$0(AsyncCleanerService.java:55)
at
java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748) {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
