[
https://issues.apache.org/jira/browse/HUDI-3393?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
sivabalan narayanan closed HUDI-3393.
-------------------------------------
Resolution: Fixed
> Deltastreamer fails during replace commit rollback and cannot proceed
> ---------------------------------------------------------------------
>
> Key: HUDI-3393
> URL: https://issues.apache.org/jira/browse/HUDI-3393
> Project: Apache Hudi
> Issue Type: Task
> Components: deltastreamer
> Reporter: sivabalan narayanan
> Assignee: sivabalan narayanan
> Priority: Critical
> Fix For: 0.11.0
>
> Attachments: image.png
>
> Original Estimate: 3h
> Remaining Estimate: 3h
>
> When a clustering commit failed mid-way and deltastreamer restarted.
> there could be one delta commit that could have succeeded. And so next time
> when clustering was attempted, it tries to rollback the older pending
> clustering and fails with "found commits after X, rollback greater commits
> first".
>
> {code:java}
> [2022-02-08 09:01:55,808] {{ssh_operator.py:143}} INFO - 22/02/08 09:01:55
> INFO Client: client token: N/A diagnostics: User class threw
> exception: org.apache.hudi.exception.HoodieRollbackException: Found commits
> after time :20220208081622, please rollback greater commits first at
> org.apache.hudi.table.action.rollback.BaseRollbackActionExecutor.validateRollbackCommitSequence(BaseRollbackActionExecutor.java:148)
> at
> org.apache.hudi.table.action.rollback.BaseRollbackActionExecutor.doRollbackAndGetStats(BaseRollbackActionExecutor.java:179)
> at
> org.apache.hudi.table.action.rollback.BaseRollbackActionExecutor.execute(BaseRollbackActionExecutor.java:102)
> at
> org.apache.hudi.table.HoodieSparkCopyOnWriteTable.rollback(HoodieSparkCopyOnWriteTable.java:237)
> at
> org.apache.hudi.client.AbstractHoodieWriteClient.rollbackInflightClustering(AbstractHoodieWriteClient.java:971)
> at
> org.apache.hudi.client.SparkRDDWriteClient.cluster(SparkRDDWriteClient.java:341)
> at
> org.apache.hudi.client.AbstractHoodieWriteClient.lambda$runAnyPendingClustering$1(AbstractHoodieWriteClient.java:475)
> at
> java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1384)
> at
> java.util.stream.ReferencePipeline$Head.forEach(ReferencePipeline.java:647)
> at
> org.apache.hudi.client.AbstractHoodieWriteClient.runAnyPendingClustering(AbstractHoodieWriteClient.java:471)
> at
> org.apache.hudi.client.AbstractHoodieWriteClient.runTableServicesInline(AbstractHoodieWriteClient.java:453)
> at
> org.apache.hudi.client.AbstractHoodieWriteClient.commitStats(AbstractHoodieWriteClient.java:194)
> at
> org.apache.hudi.client.SparkRDDWriteClient.commit(SparkRDDWriteClient.java:121)
> at
> org.apache.hudi.client.SparkRDDWriteClient.commit(SparkRDDWriteClient.java:72)
> at
> org.apache.hudi.client.AbstractHoodieWriteClient.commit(AbstractHoodieWriteClient.java:162)
> at
> org.apache.hudi.utilities.deltastreamer.DeltaSync.writeToSink(DeltaSync.java:465)
> at
> org.apache.hudi.utilities.deltastreamer.DeltaSync.syncOnce(DeltaSync.java:282)
> at
> org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.lambda$sync$2(HoodieDeltaStreamer.java:172)
> at org.apache.hudi.common.util.Option.ifPresent(Option.java:96) at
> org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.sync(HoodieDeltaStreamer.java:170)
> at
> org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.main(HoodieDeltaStreamer.java:472)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498) at
> org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:735)
> ApplicationMaster host: ip-172-19-25-155.eu-west-1.compute.internal
> ApplicationMaster RPC port: 38211 queue: default start time:
> 1644310580365 final status: FAILED tracking URL:
> http://ip-172-19-25-101.eu-west-1.compute.internal:20888/proxy/application_1643535866994_31483/
> user: root22/02/08 09:01:55 ERROR Client: Application diagnostics message:
> User class threw exception:
> org.apache.hudi.exception.HoodieRollbackException: Found commits after time
> :20220208081622, please rollback greater commits first at
> org.apache.hudi.table.action.rollback.BaseRollbackActionExecutor.validateRollbackCommitSequence(BaseRollbackActionExecutor.java:148)
> at
> org.apache.hudi.table.action.rollback.BaseRollbackActionExecutor.doRollbackAndGetStats(BaseRollbackActionExecutor.java:179)
> at
> org.apache.hudi.table.action.rollback.BaseRollbackActionExecutor.execute(BaseRollbackActionExecutor.java:102)
> at
> org.apache.hudi.table.HoodieSparkCopyOnWriteTable.rollback(HoodieSparkCopyOnWriteTable.java:237)
> at
> org.apache.hudi.client.AbstractHoodieWriteClient.rollbackInflightClustering(AbstractHoodieWriteClient.java:971)
> at
> org.apache.hudi.client.SparkRDDWriteClient.cluster(SparkRDDWriteClient.java:341)
> at
> org.apache.hudi.client.AbstractHoodieWriteClient.lambda$runAnyPendingClustering$1(AbstractHoodieWriteClient.java:475)
> at
> java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1384)
> at
> java.util.stream.ReferencePipeline$Head.forEach(ReferencePipeline.java:647)
> at
> org.apache.hudi.client.AbstractHoodieWriteClient.runAnyPendingClustering(AbstractHoodieWriteClient.java:471)
> at
> org.apache.hudi.client.AbstractHoodieWriteClient.runTableServicesInline(AbstractHoodieWriteClient.java:453)
> at
> org.apache.hudi.client.AbstractHoodieWriteClient.commitStats(AbstractHoodieWriteClient.java:194)
> at
> org.apache.hudi.client.SparkRDDWriteClient.commit(SparkRDDWriteClient.java:121)
> at
> org.apache.hudi.client.SparkRDDWriteClient.commit(SparkRDDWriteClient.java:72)
> at
> org.apache.hudi.client.AbstractHoodieWriteClient.commit(AbstractHoodieWriteClient.java:162)
> at
> org.apache.hudi.utilities.deltastreamer.DeltaSync.writeToSink(DeltaSync.java:465)
> at
> org.apache.hudi.utilities.deltastreamer.DeltaSync.syncOnce(DeltaSync.java:282)
> at
> org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.lambda$sync$2(HoodieDeltaStreamer.java:172)
> at org.apache.hudi.common.util.Option.ifPresent(Option.java:96) at
> org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.sync(HoodieDeltaStreamer.java:170)
> at
> org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.main(HoodieDeltaStreamer.java:472)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498) at
> org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:735)
> {code}
--
This message was sent by Atlassian Jira
(v8.20.1#820001)
