[
https://issues.apache.org/jira/browse/HUDI-3641?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Ethan Guo updated HUDI-3641:
----------------------------
Component/s: cleaning
compaction
table-service
> Timeline server have different last known instant in the timeline from client
> -----------------------------------------------------------------------------
>
> Key: HUDI-3641
> URL: https://issues.apache.org/jira/browse/HUDI-3641
> Project: Apache Hudi
> Issue Type: Bug
> Components: cleaning, compaction, table-service
> Reporter: Ethan Guo
> Priority: Major
>
> Hudi 0.9.0 Multi-writer scenario, one writer doing ingesting with
> Deltastreamer continuous mode, MOR, inserts, async compaction and cleaning
> (partitions under 2022/1, 2022/2), another writer with Spark datasource doing
> backfills to different partitions (2021/12). At some point, the backfill job
> hits the following exception, due to timeline server having different last
> known instant in the timeline from client. We need triage if this still
> happens in latest master.
> {code:java}
> 22/03/14 12:23:49 ERROR RequestHandler: Got runtime exception servicing
> request
> partition=2021%2F12%2F1&maxinstant=20220314122055&includependingcompaction=false&basepath=file%3A%2FUsers%2Fethan%2FWork%2Fscripts%2Fmt_rollout_testing%2Fdeploy_c_multi_writer%2Fc5_mor_09mt_011mt%2Ftest_table&lastinstantts=20220314122055&timelinehash=9fa9326a9f7d25818e648df109d89ef5ddde53021dbc5b4bd234ff8e0d0429e8
> java.lang.IllegalArgumentException: Last known instant from client was
> 20220314122055 but server has the following timeline
> [[20220314122055__deltacommit__COMPLETED],
> [20220314122143__deltacommit__COMPLETED]]
> at
> org.apache.hudi.common.util.ValidationUtils.checkArgument(ValidationUtils.java:40)
> at
> org.apache.hudi.timeline.service.RequestHandler$ViewHandler.handle(RequestHandler.java:510)
> at io.javalin.security.SecurityUtil.noopAccessManager(SecurityUtil.kt:22)
> at io.javalin.Javalin.lambda$addHandler$0(Javalin.java:606)
> at io.javalin.core.JavalinServlet$service$2$1.invoke(JavalinServlet.kt:46)
> at io.javalin.core.JavalinServlet$service$2$1.invoke(JavalinServlet.kt:17)
> at io.javalin.core.JavalinServlet$service$1.invoke(JavalinServlet.kt:143)
> at io.javalin.core.JavalinServlet$service$2.invoke(JavalinServlet.kt:41)
> at io.javalin.core.JavalinServlet.service(JavalinServlet.kt:107)
> at
> io.javalin.core.util.JettyServerUtil$initialize$httpHandler$1.doHandle(JettyServerUtil.kt:72)
> at
> org.apache.hudi.org.apache.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
> at
> org.apache.hudi.org.apache.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
> at
> org.apache.hudi.org.apache.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1668)
> at
> org.apache.hudi.org.apache.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
> at
> org.apache.hudi.org.apache.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
> at
> org.apache.hudi.org.apache.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
> at
> org.apache.hudi.org.apache.jetty.server.handler.HandlerList.handle(HandlerList.java:61)
> at
> org.apache.hudi.org.apache.jetty.server.handler.StatisticsHandler.handle(StatisticsHandler.java:174)
> at
> org.apache.hudi.org.apache.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
> at org.apache.hudi.org.apache.jetty.server.Server.handle(Server.java:502)
> at
> org.apache.hudi.org.apache.jetty.server.HttpChannel.handle(HttpChannel.java:370)
> at
> org.apache.hudi.org.apache.jetty.server.HttpConnection.onFillable(HttpConnection.java:267)
> at
> org.apache.hudi.org.apache.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
> at
> org.apache.hudi.org.apache.jetty.io.FillInterest.fillable(FillInterest.java:103)
> at
> org.apache.hudi.org.apache.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)
> at
> org.apache.hudi.org.apache.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
> at
> org.apache.hudi.org.apache.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
> at java.lang.Thread.run(Thread.java:748)
> 22/03/14 12:23:49 WARN ExceptionMapper: Uncaught exception
> java.lang.IllegalArgumentException: Last known instant from client was
> 20220314122055 but server has the following timeline
> [[20220314122055__deltacommit__COMPLETED],
> [20220314122143__deltacommit__COMPLETED]]
> at
> org.apache.hudi.common.util.ValidationUtils.checkArgument(ValidationUtils.java:40)
> at
> org.apache.hudi.timeline.service.RequestHandler$ViewHandler.handle(RequestHandler.java:510)
> at io.javalin.security.SecurityUtil.noopAccessManager(SecurityUtil.kt:22)
> at io.javalin.Javalin.lambda$addHandler$0(Javalin.java:606)
> at io.javalin.core.JavalinServlet$service$2$1.invoke(JavalinServlet.kt:46)
> at io.javalin.core.JavalinServlet$service$2$1.invoke(JavalinServlet.kt:17)
> at io.javalin.core.JavalinServlet$service$1.invoke(JavalinServlet.kt:143)
> at io.javalin.core.JavalinServlet$service$2.invoke(JavalinServlet.kt:41)
> at io.javalin.core.JavalinServlet.service(JavalinServlet.kt:107)
> at
> io.javalin.core.util.JettyServerUtil$initialize$httpHandler$1.doHandle(JettyServerUtil.kt:72)
> at
> org.apache.hudi.org.apache.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
> at
> org.apache.hudi.org.apache.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
> at
> org.apache.hudi.org.apache.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1668)
> at
> org.apache.hudi.org.apache.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
> at
> org.apache.hudi.org.apache.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
> at
> org.apache.hudi.org.apache.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
> at
> org.apache.hudi.org.apache.jetty.server.handler.HandlerList.handle(HandlerList.java:61)
> at
> org.apache.hudi.org.apache.jetty.server.handler.StatisticsHandler.handle(StatisticsHandler.java:174)
> at
> org.apache.hudi.org.apache.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
> at org.apache.hudi.org.apache.jetty.server.Server.handle(Server.java:502)
> at
> org.apache.hudi.org.apache.jetty.server.HttpChannel.handle(HttpChannel.java:370)
> at
> org.apache.hudi.org.apache.jetty.server.HttpConnection.onFillable(HttpConnection.java:267)
> at
> org.apache.hudi.org.apache.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
> at
> org.apache.hudi.org.apache.jetty.io.FillInterest.fillable(FillInterest.java:103)
> at
> org.apache.hudi.org.apache.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)
> at
> org.apache.hudi.org.apache.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
> at
> org.apache.hudi.org.apache.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
> at java.lang.Thread.run(Thread.java:748)
> 22/03/14 12:23:49 ERROR PriorityBasedFileSystemView: Got error running
> preferred function. Trying secondary
> org.apache.hudi.exception.HoodieRemoteException: Server Error
> at
> org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.getLatestFileSlicesBeforeOrOn(RemoteHoodieTableFileSystemView.java:324)
> at
> org.apache.hudi.common.table.view.PriorityBasedFileSystemView.execute(PriorityBasedFileSystemView.java:113)
> at
> org.apache.hudi.common.table.view.PriorityBasedFileSystemView.getLatestFileSlicesBeforeOrOn(PriorityBasedFileSystemView.java:172)
> at
> org.apache.hudi.table.action.deltacommit.SparkUpsertDeltaCommitPartitioner.getSmallFiles(SparkUpsertDeltaCommitPartitioner.java:72)
> at
> org.apache.hudi.table.action.commit.UpsertPartitioner.lambda$getSmallFilesForPartitions$f1d92f9e$1(UpsertPartitioner.java:256)
> at
> org.apache.spark.api.java.JavaPairRDD$.$anonfun$pairFunToScalaFun$1(JavaPairRDD.scala:1073)
> at scala.collection.Iterator$$anon$10.next(Iterator.scala:459)
> at scala.collection.Iterator.foreach(Iterator.scala:941)
> at scala.collection.Iterator.foreach$(Iterator.scala:941)
> at scala.collection.AbstractIterator.foreach(Iterator.scala:1429)
> at scala.collection.generic.Growable.$plus$plus$eq(Growable.scala:62)
> at scala.collection.generic.Growable.$plus$plus$eq$(Growable.scala:53)
> at
> scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:105)
> at
> scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:49)
> at scala.collection.TraversableOnce.to(TraversableOnce.scala:315)
> at scala.collection.TraversableOnce.to$(TraversableOnce.scala:313)
> at scala.collection.AbstractIterator.to(Iterator.scala:1429)
> at scala.collection.TraversableOnce.toBuffer(TraversableOnce.scala:307)
> at scala.collection.TraversableOnce.toBuffer$(TraversableOnce.scala:307)
> at scala.collection.AbstractIterator.toBuffer(Iterator.scala:1429)
> at scala.collection.TraversableOnce.toArray(TraversableOnce.scala:294)
> at scala.collection.TraversableOnce.toArray$(TraversableOnce.scala:288)
> at scala.collection.AbstractIterator.toArray(Iterator.scala:1429)
> at org.apache.spark.rdd.RDD.$anonfun$collect$2(RDD.scala:1030)
> at
> org.apache.spark.SparkContext.$anonfun$runJob$5(SparkContext.scala:2236)
> at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
> at org.apache.spark.scheduler.Task.run(Task.scala:131)
> at
> org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
> at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
> at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> Caused by: org.apache.http.client.HttpResponseException: Server Error
> at
> org.apache.http.impl.client.AbstractResponseHandler.handleResponse(AbstractResponseHandler.java:70)
> at org.apache.http.client.fluent.Response.handleResponse(Response.java:90)
> at org.apache.http.client.fluent.Response.returnContent(Response.java:97)
> at
> org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.executeRequest(RemoteHoodieTableFileSystemView.java:179)
> at
> org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.getLatestFileSlicesBeforeOrOn(RemoteHoodieTableFileSystemView.java:320)
> ... 32 more {code}
>
>
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
