[
https://issues.apache.org/jira/browse/HUDI-5817?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Ethan Guo updated HUDI-5817:
----------------------------
Description:
Even though the async indexer metadata writer is configured to use LAZY failed
write cleaning policy, there is other logic that can potentially roll back the
delta commits from regular metadata writer, e.g., since heartbeats are disabled
for regular MDT writes, failed write cleaning is going to rollback commits
regardless. We need to fix this so that the async indexer metadata writer does
not touch other delta commits in the MDT.
This can cause the following test to be flaky:
{code:java}
2023-02-16T13:46:06.1573775Z [ERROR] Tests run: 113, Failures: 0, Errors: 1,
Skipped: 2, Time elapsed: 3,518.191 s <<< FAILURE! - in
org.apache.hudi.utilities.deltastreamer.TestHoodieDeltaStreamer
2023-02-16T13:46:06.1576031Z [ERROR] testHoodieIndexer{HoodieRecordType}[2]
Time elapsed: 79.838 s <<< ERROR!
2023-02-16T13:46:06.1576937Z java.util.concurrent.ExecutionException:
java.lang.RuntimeException: org.apache.hudi.exception.HoodieException
2023-02-16T13:46:06.1577820Z at
java.util.concurrent.FutureTask.report(FutureTask.java:122)
2023-02-16T13:46:06.1578597Z at
java.util.concurrent.FutureTask.get(FutureTask.java:192)
2023-02-16T13:46:06.1579536Z at
org.apache.hudi.utilities.deltastreamer.TestHoodieDeltaStreamer.deltaStreamerTestRunner(TestHoodieDeltaStreamer.java:901)
2023-02-16T13:46:06.1580628Z at
org.apache.hudi.utilities.deltastreamer.TestHoodieDeltaStreamer.deltaStreamerTestRunner(TestHoodieDeltaStreamer.java:884)
2023-02-16T13:46:06.1581740Z at
org.apache.hudi.utilities.deltastreamer.TestHoodieDeltaStreamer.deltaStreamerTestRunner(TestHoodieDeltaStreamer.java:929)
2023-02-16T13:46:06.1582838Z at
org.apache.hudi.utilities.deltastreamer.TestHoodieDeltaStreamer.testHoodieIndexer(TestHoodieDeltaStreamer.java:1163)
2023-02-16T13:46:06.1583757Z at
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
2023-02-16T13:46:06.1584522Z at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
2023-02-16T13:46:06.1585420Z at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
2023-02-16T13:46:06.1586228Z at
java.lang.reflect.Method.invoke(Method.java:498)
2023-02-16T13:46:06.1587044Z at
org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
2023-02-16T13:46:06.1587939Z at
org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
2023-02-16T13:46:06.1594100Z at
org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
2023-02-16T13:46:06.1595968Z at
org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
2023-02-16T13:46:06.1597065Z at
org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestableMethod(TimeoutExtension.java:140)
2023-02-16T13:46:06.1598245Z at
org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestTemplateMethod(TimeoutExtension.java:92)
2023-02-16T13:46:06.1599109Z at
org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
2023-02-16T13:46:06.1599961Z at
org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
2023-02-16T13:46:06.1600807Z at
org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
2023-02-16T13:46:06.1601654Z at
org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
2023-02-16T13:46:06.1602480Z at
org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
2023-02-16T13:46:06.1603309Z at
org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
2023-02-16T13:46:06.1604064Z at
org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
2023-02-16T13:46:06.1604757Z at
org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
2023-02-16T13:46:06.1605552Z at
org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeTestMethod$6(TestMethodTestDescriptor.java:210)
2023-02-16T13:46:06.1606381Z at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
2023-02-16T13:46:06.1607176Z at
org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeTestMethod(TestMethodTestDescriptor.java:206)
2023-02-16T13:46:06.1607988Z at
org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:131)
2023-02-16T13:46:06.1608891Z at
org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:65)
2023-02-16T13:46:06.1610205Z at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
2023-02-16T13:46:06.1611066Z at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
2023-02-16T13:46:06.1611875Z at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
2023-02-16T13:46:06.1612589Z at
org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
2023-02-16T13:46:06.1613320Z at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
2023-02-16T13:46:06.1614132Z at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
2023-02-16T13:46:06.1614898Z at
org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
2023-02-16T13:46:06.1615656Z at
org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
2023-02-16T13:46:06.1616533Z at
org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:32)
2023-02-16T13:46:06.1617479Z at
org.junit.platform.engine.support.hierarchical.NodeTestTask$DefaultDynamicTestExecutor.execute(NodeTestTask.java:212)
2023-02-16T13:46:06.1618327Z at
org.junit.platform.engine.support.hierarchical.NodeTestTask$DefaultDynamicTestExecutor.execute(NodeTestTask.java:192)
2023-02-16T13:46:06.1619168Z at
org.junit.jupiter.engine.descriptor.TestTemplateTestDescriptor.execute(TestTemplateTestDescriptor.java:139)
2023-02-16T13:46:06.1620011Z at
org.junit.jupiter.engine.descriptor.TestTemplateTestDescriptor.lambda$execute$2(TestTemplateTestDescriptor.java:107)
2023-02-16T13:46:06.1620740Z at
java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183)
2023-02-16T13:46:06.1621378Z at
java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
2023-02-16T13:46:06.1623243Z at
java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:175)
2023-02-16T13:46:06.1623930Z at
java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
2023-02-16T13:46:06.1624573Z at
java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183)
2023-02-16T13:46:06.1625208Z at
java.util.stream.ReferencePipeline$11$1.accept(ReferencePipeline.java:440)
2023-02-16T13:46:06.1625833Z at
java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
2023-02-16T13:46:06.1626464Z at
java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
2023-02-16T13:46:06.1627095Z at
java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
2023-02-16T13:46:06.1627708Z at
java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183)
2023-02-16T13:46:06.1628338Z at
java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
2023-02-16T13:46:06.1629056Z at
java.util.Iterator.forEachRemaining(Iterator.java:116)
2023-02-16T13:46:06.1629666Z at
java.util.Spliterators$IteratorSpliterator.forEachRemaining(Spliterators.java:1801)
2023-02-16T13:46:06.1630323Z at
java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:482)
2023-02-16T13:46:06.1630960Z at
java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
2023-02-16T13:46:06.1631599Z at
java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150)
2023-02-16T13:46:06.1632269Z at
java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173)
2023-02-16T13:46:06.1632922Z at
java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
2023-02-16T13:46:06.1633525Z at
java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:485)
2023-02-16T13:46:06.1634150Z at
java.util.stream.ReferencePipeline$7$1.accept(ReferencePipeline.java:272)
2023-02-16T13:46:06.1635825Z at
java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
2023-02-16T13:46:06.1636458Z at
java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
2023-02-16T13:46:06.1637102Z at
java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
2023-02-16T13:46:06.1637751Z at
java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1384)
2023-02-16T13:46:06.1638378Z at
java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:482)
2023-02-16T13:46:06.1639012Z at
java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
2023-02-16T13:46:06.1639664Z at
java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150)
2023-02-16T13:46:06.1640320Z at
java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173)
2023-02-16T13:46:06.1640972Z at
java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
2023-02-16T13:46:06.1641589Z at
java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:485)
2023-02-16T13:46:06.1642206Z at
java.util.stream.ReferencePipeline$7$1.accept(ReferencePipeline.java:272)
2023-02-16T13:46:06.1642854Z at
java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1384)
2023-02-16T13:46:06.1643488Z at
java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:482)
2023-02-16T13:46:06.1644133Z at
java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
2023-02-16T13:46:06.1644786Z at
java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150)
2023-02-16T13:46:06.1645462Z at
java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173)
2023-02-16T13:46:06.1646097Z at
java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
2023-02-16T13:46:06.1646717Z at
java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:485)
2023-02-16T13:46:06.1647434Z at
org.junit.jupiter.engine.descriptor.TestTemplateTestDescriptor.execute(TestTemplateTestDescriptor.java:107)
2023-02-16T13:46:06.1648237Z at
org.junit.jupiter.engine.descriptor.TestTemplateTestDescriptor.execute(TestTemplateTestDescriptor.java:42)
2023-02-16T13:46:06.1649207Z at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
2023-02-16T13:46:06.1650019Z at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
2023-02-16T13:46:06.1650811Z at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
2023-02-16T13:46:06.1651547Z at
org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
2023-02-16T13:46:06.1652275Z at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
2023-02-16T13:46:06.1653086Z at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
2023-02-16T13:46:06.1653857Z at
org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
2023-02-16T13:46:06.1654614Z at
org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
2023-02-16T13:46:06.1655234Z at
java.util.ArrayList.forEach(ArrayList.java:1259)
2023-02-16T13:46:06.1655994Z at
org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
2023-02-16T13:46:06.1656929Z at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
2023-02-16T13:46:06.1657733Z at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
2023-02-16T13:46:06.1658520Z at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
2023-02-16T13:46:06.1659252Z at
org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
2023-02-16T13:46:06.1660059Z at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
2023-02-16T13:46:06.1660849Z at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
2023-02-16T13:46:06.1661623Z at
org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
2023-02-16T13:46:06.1662367Z at
org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
2023-02-16T13:46:06.1662967Z at
java.util.ArrayList.forEach(ArrayList.java:1259)
2023-02-16T13:46:06.1663737Z at
org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
2023-02-16T13:46:06.1664670Z at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
2023-02-16T13:46:06.1665480Z at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
2023-02-16T13:46:06.1666268Z at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
2023-02-16T13:46:06.1667003Z at
org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
2023-02-16T13:46:06.1667736Z at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
2023-02-16T13:46:06.1668522Z at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
2023-02-16T13:46:06.1670214Z at
org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
2023-02-16T13:46:06.1670985Z at
org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
2023-02-16T13:46:06.1671842Z at
org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:32)
2023-02-16T13:46:06.1672789Z at
org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutor.execute(HierarchicalTestExecutor.java:57)
2023-02-16T13:46:06.1674085Z at
org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine.execute(HierarchicalTestEngine.java:51)
2023-02-16T13:46:06.1674890Z at
org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:108)
2023-02-16T13:46:06.1675698Z at
org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:88)
2023-02-16T13:46:06.1676783Z at
org.junit.platform.launcher.core.EngineExecutionOrchestrator.lambda$execute$0(EngineExecutionOrchestrator.java:54)
2023-02-16T13:46:06.1677657Z at
org.junit.platform.launcher.core.EngineExecutionOrchestrator.withInterceptedStreams(EngineExecutionOrchestrator.java:67)
2023-02-16T13:46:06.1678489Z at
org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:52)
2023-02-16T13:46:06.1679238Z at
org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:96)
2023-02-16T13:46:06.1679917Z at
org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:75)
2023-02-16T13:46:06.1681491Z at
org.apache.maven.surefire.junitplatform.JUnitPlatformProvider.invokeAllTests(JUnitPlatformProvider.java:150)
2023-02-16T13:46:06.1682401Z at
org.apache.maven.surefire.junitplatform.JUnitPlatformProvider.invoke(JUnitPlatformProvider.java:124)
2023-02-16T13:46:06.1684238Z at
org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:384)
2023-02-16T13:46:06.1684986Z at
org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:345)
2023-02-16T13:46:06.1686438Z at
org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:126)
2023-02-16T13:46:06.1687115Z at
org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:418)
2023-02-16T13:46:06.1687704Z Caused by: java.lang.RuntimeException:
org.apache.hudi.exception.HoodieException
2023-02-16T13:46:06.1688684Z at
org.apache.hudi.utilities.deltastreamer.TestHoodieDeltaStreamer.lambda$deltaStreamerTestRunner$9(TestHoodieDeltaStreamer.java:893)
2023-02-16T13:46:06.1689478Z at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
2023-02-16T13:46:06.1690064Z at
java.util.concurrent.FutureTask.run(FutureTask.java:266)
2023-02-16T13:46:06.1690677Z at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
2023-02-16T13:46:06.1691351Z at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
2023-02-16T13:46:06.1691893Z at java.lang.Thread.run(Thread.java:750)
2023-02-16T13:46:06.1692434Z Caused by:
org.apache.hudi.exception.HoodieException:
org.apache.hudi.exception.HoodieException
2023-02-16T13:46:06.1694361Z at
org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.lambda$sync$1(HoodieDeltaStreamer.java:197)
2023-02-16T13:46:06.1695072Z at
org.apache.hudi.common.util.Option.ifPresent(Option.java:97)
2023-02-16T13:46:06.1696111Z at
org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.sync(HoodieDeltaStreamer.java:192)
2023-02-16T13:46:06.1696972Z at
org.apache.hudi.utilities.deltastreamer.TestHoodieDeltaStreamer.lambda$deltaStreamerTestRunner$9(TestHoodieDeltaStreamer.java:890)
2023-02-16T13:46:06.1698335Z ... 5 more
2023-02-16T13:46:06.1698799Z Caused by:
java.util.concurrent.ExecutionException:
org.apache.hudi.exception.HoodieException
2023-02-16T13:46:06.1699436Z at
java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:357)
2023-02-16T13:46:06.1700053Z at
java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1908)
2023-02-16T13:46:06.1700715Z at
org.apache.hudi.async.HoodieAsyncService.waitForShutdown(HoodieAsyncService.java:103)
2023-02-16T13:46:06.1701465Z at
org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.lambda$sync$1(HoodieDeltaStreamer.java:195)
2023-02-16T13:46:06.1701987Z ... 8 more
2023-02-16T13:46:06.1702354Z Caused by:
org.apache.hudi.exception.HoodieException
2023-02-16T13:46:06.1703051Z at
org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer$DeltaSyncService.lambda$startService$1(HoodieDeltaStreamer.java:758)
2023-02-16T13:46:06.1704867Z at
java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)
2023-02-16T13:46:06.1705362Z ... 3 more
2023-02-16T13:46:06.1705711Z Caused by: java.lang.IllegalArgumentException
2023-02-16T13:46:06.1706251Z at
org.apache.hudi.common.util.ValidationUtils.checkArgument(ValidationUtils.java:31)
2023-02-16T13:46:06.1706995Z at
org.apache.hudi.common.table.timeline.HoodieActiveTimeline.transitionState(HoodieActiveTimeline.java:633)
2023-02-16T13:46:06.1707847Z at
org.apache.hudi.common.table.timeline.HoodieActiveTimeline.transitionRequestedToInflight(HoodieActiveTimeline.java:698)
2023-02-16T13:46:06.1708751Z at
org.apache.hudi.table.action.commit.BaseCommitActionExecutor.saveWorkloadProfileMetadataToInflight(BaseCommitActionExecutor.java:147)
2023-02-16T13:46:06.1709792Z at
org.apache.hudi.table.action.commit.BaseSparkCommitActionExecutor.execute(BaseSparkCommitActionExecutor.java:172)
2023-02-16T13:46:06.1710733Z at
org.apache.hudi.table.action.deltacommit.SparkUpsertPreppedDeltaCommitActionExecutor.execute(SparkUpsertPreppedDeltaCommitActionExecutor.java:44)
2023-02-16T13:46:06.1712815Z at
org.apache.hudi.table.HoodieSparkMergeOnReadTable.upsertPrepped(HoodieSparkMergeOnReadTable.java:111)
2023-02-16T13:46:06.1713593Z at
org.apache.hudi.table.HoodieSparkMergeOnReadTable.upsertPrepped(HoodieSparkMergeOnReadTable.java:80)
2023-02-16T13:46:06.1714353Z at
org.apache.hudi.client.SparkRDDWriteClient.upsertPreppedRecords(SparkRDDWriteClient.java:154)
2023-02-16T13:46:06.1715155Z at
org.apache.hudi.metadata.SparkHoodieBackedTableMetadataWriter.commit(SparkHoodieBackedTableMetadataWriter.java:186)
2023-02-16T13:46:06.1716395Z at
org.apache.hudi.metadata.HoodieBackedTableMetadataWriter.processAndCommit(HoodieBackedTableMetadataWriter.java:830)
2023-02-16T13:46:06.1718036Z at
org.apache.hudi.metadata.HoodieBackedTableMetadataWriter.update(HoodieBackedTableMetadataWriter.java:897)
2023-02-16T13:46:06.1718885Z at
org.apache.hudi.client.BaseHoodieWriteClient.lambda$writeTableMetadata$1(BaseHoodieWriteClient.java:355)
2023-02-16T13:46:06.1719843Z at
org.apache.hudi.common.util.Option.ifPresent(Option.java:97)
2023-02-16T13:46:06.1720514Z at
org.apache.hudi.client.BaseHoodieWriteClient.writeTableMetadata(BaseHoodieWriteClient.java:355)
2023-02-16T13:46:06.1721234Z at
org.apache.hudi.client.BaseHoodieWriteClient.commit(BaseHoodieWriteClient.java:282)
2023-02-16T13:46:06.1722195Z at
org.apache.hudi.client.BaseHoodieWriteClient.commitStats(BaseHoodieWriteClient.java:233)
2023-02-16T13:46:06.1722912Z at
org.apache.hudi.client.SparkRDDWriteClient.commit(SparkRDDWriteClient.java:102)
2023-02-16T13:46:06.1723971Z at
org.apache.hudi.client.SparkRDDWriteClient.commit(SparkRDDWriteClient.java:61)
2023-02-16T13:46:06.1724682Z at
org.apache.hudi.client.BaseHoodieWriteClient.commit(BaseHoodieWriteClient.java:199)
2023-02-16T13:46:06.1725385Z at
org.apache.hudi.utilities.deltastreamer.DeltaSync.writeToSink(DeltaSync.java:713)
2023-02-16T13:46:06.1726061Z at
org.apache.hudi.utilities.deltastreamer.DeltaSync.syncOnce(DeltaSync.java:395)
2023-02-16T13:46:06.1726846Z at
org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer$DeltaSyncService.lambda$startService$1(HoodieDeltaStreamer.java:716)
2023-02-16T13:46:06.1727456Z ... 4 more {code}
was:Even though the async indexer metadata writer is configured to use LAZY
failed write cleaning policy, there is other logic that can potentially roll
back the delta commits from regular metadata writer, e.g., since heartbeats are
disabled for regular MDT writes, failed write cleaning is going to rollback
commits regardless. We need to fix this so that the async indexer metadata
writer does not touch other delta commits in the MDT.
> Fix async indexer metadata writer to avoid eager rollback / cleaning
> --------------------------------------------------------------------
>
> Key: HUDI-5817
> URL: https://issues.apache.org/jira/browse/HUDI-5817
> Project: Apache Hudi
> Issue Type: Improvement
> Reporter: Ethan Guo
> Assignee: Ethan Guo
> Priority: Critical
> Fix For: 0.13.1
>
>
> Even though the async indexer metadata writer is configured to use LAZY
> failed write cleaning policy, there is other logic that can potentially roll
> back the delta commits from regular metadata writer, e.g., since heartbeats
> are disabled for regular MDT writes, failed write cleaning is going to
> rollback commits regardless. We need to fix this so that the async indexer
> metadata writer does not touch other delta commits in the MDT.
>
> This can cause the following test to be flaky:
> {code:java}
> 2023-02-16T13:46:06.1573775Z [ERROR] Tests run: 113, Failures: 0, Errors: 1,
> Skipped: 2, Time elapsed: 3,518.191 s <<< FAILURE! - in
> org.apache.hudi.utilities.deltastreamer.TestHoodieDeltaStreamer
> 2023-02-16T13:46:06.1576031Z [ERROR] testHoodieIndexer{HoodieRecordType}[2]
> Time elapsed: 79.838 s <<< ERROR!
> 2023-02-16T13:46:06.1576937Z java.util.concurrent.ExecutionException:
> java.lang.RuntimeException: org.apache.hudi.exception.HoodieException
> 2023-02-16T13:46:06.1577820Z at
> java.util.concurrent.FutureTask.report(FutureTask.java:122)
> 2023-02-16T13:46:06.1578597Z at
> java.util.concurrent.FutureTask.get(FutureTask.java:192)
> 2023-02-16T13:46:06.1579536Z at
> org.apache.hudi.utilities.deltastreamer.TestHoodieDeltaStreamer.deltaStreamerTestRunner(TestHoodieDeltaStreamer.java:901)
> 2023-02-16T13:46:06.1580628Z at
> org.apache.hudi.utilities.deltastreamer.TestHoodieDeltaStreamer.deltaStreamerTestRunner(TestHoodieDeltaStreamer.java:884)
> 2023-02-16T13:46:06.1581740Z at
> org.apache.hudi.utilities.deltastreamer.TestHoodieDeltaStreamer.deltaStreamerTestRunner(TestHoodieDeltaStreamer.java:929)
> 2023-02-16T13:46:06.1582838Z at
> org.apache.hudi.utilities.deltastreamer.TestHoodieDeltaStreamer.testHoodieIndexer(TestHoodieDeltaStreamer.java:1163)
> 2023-02-16T13:46:06.1583757Z at
> sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 2023-02-16T13:46:06.1584522Z at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> 2023-02-16T13:46:06.1585420Z at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 2023-02-16T13:46:06.1586228Z at
> java.lang.reflect.Method.invoke(Method.java:498)
> 2023-02-16T13:46:06.1587044Z at
> org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
> 2023-02-16T13:46:06.1587939Z at
> org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
> 2023-02-16T13:46:06.1594100Z at
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
> 2023-02-16T13:46:06.1595968Z at
> org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
> 2023-02-16T13:46:06.1597065Z at
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestableMethod(TimeoutExtension.java:140)
> 2023-02-16T13:46:06.1598245Z at
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestTemplateMethod(TimeoutExtension.java:92)
> 2023-02-16T13:46:06.1599109Z at
> org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
> 2023-02-16T13:46:06.1599961Z at
> org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
> 2023-02-16T13:46:06.1600807Z at
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
> 2023-02-16T13:46:06.1601654Z at
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
> 2023-02-16T13:46:06.1602480Z at
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
> 2023-02-16T13:46:06.1603309Z at
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
> 2023-02-16T13:46:06.1604064Z at
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
> 2023-02-16T13:46:06.1604757Z at
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
> 2023-02-16T13:46:06.1605552Z at
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeTestMethod$6(TestMethodTestDescriptor.java:210)
> 2023-02-16T13:46:06.1606381Z at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> 2023-02-16T13:46:06.1607176Z at
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeTestMethod(TestMethodTestDescriptor.java:206)
> 2023-02-16T13:46:06.1607988Z at
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:131)
> 2023-02-16T13:46:06.1608891Z at
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:65)
> 2023-02-16T13:46:06.1610205Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
> 2023-02-16T13:46:06.1611066Z at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> 2023-02-16T13:46:06.1611875Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> 2023-02-16T13:46:06.1612589Z at
> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> 2023-02-16T13:46:06.1613320Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> 2023-02-16T13:46:06.1614132Z at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> 2023-02-16T13:46:06.1614898Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> 2023-02-16T13:46:06.1615656Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> 2023-02-16T13:46:06.1616533Z at
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:32)
> 2023-02-16T13:46:06.1617479Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask$DefaultDynamicTestExecutor.execute(NodeTestTask.java:212)
> 2023-02-16T13:46:06.1618327Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask$DefaultDynamicTestExecutor.execute(NodeTestTask.java:192)
> 2023-02-16T13:46:06.1619168Z at
> org.junit.jupiter.engine.descriptor.TestTemplateTestDescriptor.execute(TestTemplateTestDescriptor.java:139)
> 2023-02-16T13:46:06.1620011Z at
> org.junit.jupiter.engine.descriptor.TestTemplateTestDescriptor.lambda$execute$2(TestTemplateTestDescriptor.java:107)
> 2023-02-16T13:46:06.1620740Z at
> java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183)
> 2023-02-16T13:46:06.1621378Z at
> java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
> 2023-02-16T13:46:06.1623243Z at
> java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:175)
> 2023-02-16T13:46:06.1623930Z at
> java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
> 2023-02-16T13:46:06.1624573Z at
> java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183)
> 2023-02-16T13:46:06.1625208Z at
> java.util.stream.ReferencePipeline$11$1.accept(ReferencePipeline.java:440)
> 2023-02-16T13:46:06.1625833Z at
> java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
> 2023-02-16T13:46:06.1626464Z at
> java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
> 2023-02-16T13:46:06.1627095Z at
> java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
> 2023-02-16T13:46:06.1627708Z at
> java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183)
> 2023-02-16T13:46:06.1628338Z at
> java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
> 2023-02-16T13:46:06.1629056Z at
> java.util.Iterator.forEachRemaining(Iterator.java:116)
> 2023-02-16T13:46:06.1629666Z at
> java.util.Spliterators$IteratorSpliterator.forEachRemaining(Spliterators.java:1801)
> 2023-02-16T13:46:06.1630323Z at
> java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:482)
> 2023-02-16T13:46:06.1630960Z at
> java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
> 2023-02-16T13:46:06.1631599Z at
> java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150)
> 2023-02-16T13:46:06.1632269Z at
> java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173)
> 2023-02-16T13:46:06.1632922Z at
> java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
> 2023-02-16T13:46:06.1633525Z at
> java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:485)
> 2023-02-16T13:46:06.1634150Z at
> java.util.stream.ReferencePipeline$7$1.accept(ReferencePipeline.java:272)
> 2023-02-16T13:46:06.1635825Z at
> java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
> 2023-02-16T13:46:06.1636458Z at
> java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
> 2023-02-16T13:46:06.1637102Z at
> java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
> 2023-02-16T13:46:06.1637751Z at
> java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1384)
> 2023-02-16T13:46:06.1638378Z at
> java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:482)
> 2023-02-16T13:46:06.1639012Z at
> java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
> 2023-02-16T13:46:06.1639664Z at
> java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150)
> 2023-02-16T13:46:06.1640320Z at
> java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173)
> 2023-02-16T13:46:06.1640972Z at
> java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
> 2023-02-16T13:46:06.1641589Z at
> java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:485)
> 2023-02-16T13:46:06.1642206Z at
> java.util.stream.ReferencePipeline$7$1.accept(ReferencePipeline.java:272)
> 2023-02-16T13:46:06.1642854Z at
> java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1384)
> 2023-02-16T13:46:06.1643488Z at
> java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:482)
> 2023-02-16T13:46:06.1644133Z at
> java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
> 2023-02-16T13:46:06.1644786Z at
> java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150)
> 2023-02-16T13:46:06.1645462Z at
> java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173)
> 2023-02-16T13:46:06.1646097Z at
> java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
> 2023-02-16T13:46:06.1646717Z at
> java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:485)
> 2023-02-16T13:46:06.1647434Z at
> org.junit.jupiter.engine.descriptor.TestTemplateTestDescriptor.execute(TestTemplateTestDescriptor.java:107)
> 2023-02-16T13:46:06.1648237Z at
> org.junit.jupiter.engine.descriptor.TestTemplateTestDescriptor.execute(TestTemplateTestDescriptor.java:42)
> 2023-02-16T13:46:06.1649207Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
> 2023-02-16T13:46:06.1650019Z at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> 2023-02-16T13:46:06.1650811Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> 2023-02-16T13:46:06.1651547Z at
> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> 2023-02-16T13:46:06.1652275Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> 2023-02-16T13:46:06.1653086Z at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> 2023-02-16T13:46:06.1653857Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> 2023-02-16T13:46:06.1654614Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> 2023-02-16T13:46:06.1655234Z at
> java.util.ArrayList.forEach(ArrayList.java:1259)
> 2023-02-16T13:46:06.1655994Z at
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
> 2023-02-16T13:46:06.1656929Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
> 2023-02-16T13:46:06.1657733Z at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> 2023-02-16T13:46:06.1658520Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> 2023-02-16T13:46:06.1659252Z at
> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> 2023-02-16T13:46:06.1660059Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> 2023-02-16T13:46:06.1660849Z at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> 2023-02-16T13:46:06.1661623Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> 2023-02-16T13:46:06.1662367Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> 2023-02-16T13:46:06.1662967Z at
> java.util.ArrayList.forEach(ArrayList.java:1259)
> 2023-02-16T13:46:06.1663737Z at
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
> 2023-02-16T13:46:06.1664670Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
> 2023-02-16T13:46:06.1665480Z at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> 2023-02-16T13:46:06.1666268Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> 2023-02-16T13:46:06.1667003Z at
> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> 2023-02-16T13:46:06.1667736Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> 2023-02-16T13:46:06.1668522Z at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> 2023-02-16T13:46:06.1670214Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> 2023-02-16T13:46:06.1670985Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> 2023-02-16T13:46:06.1671842Z at
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:32)
> 2023-02-16T13:46:06.1672789Z at
> org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutor.execute(HierarchicalTestExecutor.java:57)
> 2023-02-16T13:46:06.1674085Z at
> org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine.execute(HierarchicalTestEngine.java:51)
> 2023-02-16T13:46:06.1674890Z at
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:108)
> 2023-02-16T13:46:06.1675698Z at
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:88)
> 2023-02-16T13:46:06.1676783Z at
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.lambda$execute$0(EngineExecutionOrchestrator.java:54)
> 2023-02-16T13:46:06.1677657Z at
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.withInterceptedStreams(EngineExecutionOrchestrator.java:67)
> 2023-02-16T13:46:06.1678489Z at
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:52)
> 2023-02-16T13:46:06.1679238Z at
> org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:96)
> 2023-02-16T13:46:06.1679917Z at
> org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:75)
> 2023-02-16T13:46:06.1681491Z at
> org.apache.maven.surefire.junitplatform.JUnitPlatformProvider.invokeAllTests(JUnitPlatformProvider.java:150)
> 2023-02-16T13:46:06.1682401Z at
> org.apache.maven.surefire.junitplatform.JUnitPlatformProvider.invoke(JUnitPlatformProvider.java:124)
> 2023-02-16T13:46:06.1684238Z at
> org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:384)
> 2023-02-16T13:46:06.1684986Z at
> org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:345)
> 2023-02-16T13:46:06.1686438Z at
> org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:126)
> 2023-02-16T13:46:06.1687115Z at
> org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:418)
> 2023-02-16T13:46:06.1687704Z Caused by: java.lang.RuntimeException:
> org.apache.hudi.exception.HoodieException
> 2023-02-16T13:46:06.1688684Z at
> org.apache.hudi.utilities.deltastreamer.TestHoodieDeltaStreamer.lambda$deltaStreamerTestRunner$9(TestHoodieDeltaStreamer.java:893)
> 2023-02-16T13:46:06.1689478Z at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
> 2023-02-16T13:46:06.1690064Z at
> java.util.concurrent.FutureTask.run(FutureTask.java:266)
> 2023-02-16T13:46:06.1690677Z at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> 2023-02-16T13:46:06.1691351Z at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> 2023-02-16T13:46:06.1691893Z at java.lang.Thread.run(Thread.java:750)
> 2023-02-16T13:46:06.1692434Z Caused by:
> org.apache.hudi.exception.HoodieException:
> org.apache.hudi.exception.HoodieException
> 2023-02-16T13:46:06.1694361Z at
> org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.lambda$sync$1(HoodieDeltaStreamer.java:197)
> 2023-02-16T13:46:06.1695072Z at
> org.apache.hudi.common.util.Option.ifPresent(Option.java:97)
> 2023-02-16T13:46:06.1696111Z at
> org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.sync(HoodieDeltaStreamer.java:192)
> 2023-02-16T13:46:06.1696972Z at
> org.apache.hudi.utilities.deltastreamer.TestHoodieDeltaStreamer.lambda$deltaStreamerTestRunner$9(TestHoodieDeltaStreamer.java:890)
> 2023-02-16T13:46:06.1698335Z ... 5 more
> 2023-02-16T13:46:06.1698799Z Caused by:
> java.util.concurrent.ExecutionException:
> org.apache.hudi.exception.HoodieException
> 2023-02-16T13:46:06.1699436Z at
> java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:357)
> 2023-02-16T13:46:06.1700053Z at
> java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1908)
> 2023-02-16T13:46:06.1700715Z at
> org.apache.hudi.async.HoodieAsyncService.waitForShutdown(HoodieAsyncService.java:103)
> 2023-02-16T13:46:06.1701465Z at
> org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.lambda$sync$1(HoodieDeltaStreamer.java:195)
> 2023-02-16T13:46:06.1701987Z ... 8 more
> 2023-02-16T13:46:06.1702354Z Caused by:
> org.apache.hudi.exception.HoodieException
> 2023-02-16T13:46:06.1703051Z at
> org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer$DeltaSyncService.lambda$startService$1(HoodieDeltaStreamer.java:758)
> 2023-02-16T13:46:06.1704867Z at
> java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)
> 2023-02-16T13:46:06.1705362Z ... 3 more
> 2023-02-16T13:46:06.1705711Z Caused by: java.lang.IllegalArgumentException
> 2023-02-16T13:46:06.1706251Z at
> org.apache.hudi.common.util.ValidationUtils.checkArgument(ValidationUtils.java:31)
> 2023-02-16T13:46:06.1706995Z at
> org.apache.hudi.common.table.timeline.HoodieActiveTimeline.transitionState(HoodieActiveTimeline.java:633)
> 2023-02-16T13:46:06.1707847Z at
> org.apache.hudi.common.table.timeline.HoodieActiveTimeline.transitionRequestedToInflight(HoodieActiveTimeline.java:698)
> 2023-02-16T13:46:06.1708751Z at
> org.apache.hudi.table.action.commit.BaseCommitActionExecutor.saveWorkloadProfileMetadataToInflight(BaseCommitActionExecutor.java:147)
> 2023-02-16T13:46:06.1709792Z at
> org.apache.hudi.table.action.commit.BaseSparkCommitActionExecutor.execute(BaseSparkCommitActionExecutor.java:172)
> 2023-02-16T13:46:06.1710733Z at
> org.apache.hudi.table.action.deltacommit.SparkUpsertPreppedDeltaCommitActionExecutor.execute(SparkUpsertPreppedDeltaCommitActionExecutor.java:44)
> 2023-02-16T13:46:06.1712815Z at
> org.apache.hudi.table.HoodieSparkMergeOnReadTable.upsertPrepped(HoodieSparkMergeOnReadTable.java:111)
> 2023-02-16T13:46:06.1713593Z at
> org.apache.hudi.table.HoodieSparkMergeOnReadTable.upsertPrepped(HoodieSparkMergeOnReadTable.java:80)
> 2023-02-16T13:46:06.1714353Z at
> org.apache.hudi.client.SparkRDDWriteClient.upsertPreppedRecords(SparkRDDWriteClient.java:154)
> 2023-02-16T13:46:06.1715155Z at
> org.apache.hudi.metadata.SparkHoodieBackedTableMetadataWriter.commit(SparkHoodieBackedTableMetadataWriter.java:186)
> 2023-02-16T13:46:06.1716395Z at
> org.apache.hudi.metadata.HoodieBackedTableMetadataWriter.processAndCommit(HoodieBackedTableMetadataWriter.java:830)
> 2023-02-16T13:46:06.1718036Z at
> org.apache.hudi.metadata.HoodieBackedTableMetadataWriter.update(HoodieBackedTableMetadataWriter.java:897)
> 2023-02-16T13:46:06.1718885Z at
> org.apache.hudi.client.BaseHoodieWriteClient.lambda$writeTableMetadata$1(BaseHoodieWriteClient.java:355)
> 2023-02-16T13:46:06.1719843Z at
> org.apache.hudi.common.util.Option.ifPresent(Option.java:97)
> 2023-02-16T13:46:06.1720514Z at
> org.apache.hudi.client.BaseHoodieWriteClient.writeTableMetadata(BaseHoodieWriteClient.java:355)
> 2023-02-16T13:46:06.1721234Z at
> org.apache.hudi.client.BaseHoodieWriteClient.commit(BaseHoodieWriteClient.java:282)
> 2023-02-16T13:46:06.1722195Z at
> org.apache.hudi.client.BaseHoodieWriteClient.commitStats(BaseHoodieWriteClient.java:233)
> 2023-02-16T13:46:06.1722912Z at
> org.apache.hudi.client.SparkRDDWriteClient.commit(SparkRDDWriteClient.java:102)
> 2023-02-16T13:46:06.1723971Z at
> org.apache.hudi.client.SparkRDDWriteClient.commit(SparkRDDWriteClient.java:61)
> 2023-02-16T13:46:06.1724682Z at
> org.apache.hudi.client.BaseHoodieWriteClient.commit(BaseHoodieWriteClient.java:199)
> 2023-02-16T13:46:06.1725385Z at
> org.apache.hudi.utilities.deltastreamer.DeltaSync.writeToSink(DeltaSync.java:713)
> 2023-02-16T13:46:06.1726061Z at
> org.apache.hudi.utilities.deltastreamer.DeltaSync.syncOnce(DeltaSync.java:395)
> 2023-02-16T13:46:06.1726846Z at
> org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer$DeltaSyncService.lambda$startService$1(HoodieDeltaStreamer.java:716)
> 2023-02-16T13:46:06.1727456Z ... 4 more {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
