[
https://issues.apache.org/jira/browse/HUDI-3141?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
sivabalan narayanan updated HUDI-3141:
--------------------------------------
Sprint: Hudi 0.10.1 - 2021/01/03 (was: Hudi-Sprint-Jan-3)
> Metadata table getAllFilesInPartition() crashes with NullPointerException
> -------------------------------------------------------------------------
>
> Key: HUDI-3141
> URL: https://issues.apache.org/jira/browse/HUDI-3141
> Project: Apache Hudi
> Issue Type: Task
> Reporter: Manoj Govindassamy
> Assignee: Manoj Govindassamy
> Priority: Blocker
> Labels: pull-request-available
> Fix For: 0.10.1
>
>
> When doing getAllFilesInPartition(), metadata table crashes with
> NullPointerException. This is in the context of multiple clients requesting
> "datafiles/range/latest/", thats is, getLatestDataFilesBeforeOrOn() from the
> table.
>
> {noformat}
> java.lang.NullPointerException
> at
> org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader.processQueuedBlocksForInstant(AbstractHoodieLogRecordReader.java:422)
> at
> org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader.scan(AbstractHoodieLogRecordReader.java:238)
> at
> org.apache.hudi.metadata.HoodieMetadataMergedLogRecordReader.getRecordsByKeys(HoodieMetadataMergedLogRecordReader.java:126)
> at
> org.apache.hudi.metadata.HoodieBackedTableMetadata.readLogRecords(HoodieBackedTableMetadata.java:188)
> at
> org.apache.hudi.metadata.HoodieBackedTableMetadata.lambda$getRecordsByKeys$0(HoodieBackedTableMetadata.java:152)
> at java.util.HashMap.forEach(HashMap.java:1290)
> at
> org.apache.hudi.metadata.HoodieBackedTableMetadata.getRecordsByKeys(HoodieBackedTableMetadata.java:139)
> at
> org.apache.hudi.metadata.HoodieBackedTableMetadata.getRecordByKey(HoodieBackedTableMetadata.java:128)
> at
> org.apache.hudi.metadata.BaseTableMetadata.fetchAllFilesInPartition(BaseTableMetadata.java:285)
> at
> org.apache.hudi.metadata.BaseTableMetadata.getAllFilesInPartition(BaseTableMetadata.java:131)
> at
> org.apache.hudi.metadata.HoodieMetadataFileSystemView.listPartition(HoodieMetadataFileSystemView.java:65)
> at
> org.apache.hudi.common.table.view.AbstractTableFileSystemView.lambda$ensurePartitionLoadedCorrectly$9(AbstractTableFileSystemView.java:304)
> at
> java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1660)
> at
> org.apache.hudi.common.table.view.AbstractTableFileSystemView.ensurePartitionLoadedCorrectly(AbstractTableFileSystemView.java:295)
> at
> org.apache.hudi.common.table.view.AbstractTableFileSystemView.getLatestBaseFilesBeforeOrOn(AbstractTableFileSystemView.java:489)
> at
> org.apache.hudi.timeline.service.handlers.BaseFileHandler.getLatestDataFilesBeforeOrOn(BaseFileHandler.java:60)
> at
> org.apache.hudi.timeline.service.RequestHandler.lambda$registerDataFilesAPI$6(RequestHandler.java:268)
> at
> org.apache.hudi.timeline.service.RequestHandler$ViewHandler.handle(RequestHandler.java:497)
> at io.javalin.security.SecurityUtil.noopAccessManager(SecurityUtil.kt:22)
> at io.javalin.Javalin.lambda$addHandler$0(Javalin.java:606)
> at io.javalin.core.JavalinServlet$service$2$1.invoke(JavalinServlet.kt:46)
> at io.javalin.core.JavalinServlet$service$2$1.invoke(JavalinServlet.kt:17)
> at io.javalin.core.JavalinServlet$service$1.invoke(JavalinServlet.kt:143)
> at io.javalin.core.JavalinServlet$service$2.invoke(JavalinServlet.kt:41)
> at io.javalin.core.JavalinServlet.service(JavalinServlet.kt:107)
> at
> io.javalin.core.util.JettyServerUtil$initialize$httpHandler$1.doHandle(JettyServerUtil.kt:72)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
> at
> org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
> at
> org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1668)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
> at
> org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
> at
> org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:61)
> at
> org.eclipse.jetty.server.handler.StatisticsHandler.handle(StatisticsHandler.java:174)
> at
> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
> at org.eclipse.jetty.server.Server.handle(Server.java:502)
> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:370)
> at
> org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:267)
> at
> org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
> at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)
> at
> org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333)
> at
> org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310)
> at
> org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
> at
> org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126)
> at
> org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366)
> at
> org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
> at
> org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
> at java.lang.Thread.run(Thread.java:748)
> 24424 [qtp1929693831-818] ERROR
> org.apache.hudi.timeline.service.RequestHandler - Got runtime exception
> servicing request
> partition=2015%2F03%2F16&maxinstant=0000003&basepath=%2Fvar%2Ffolders%2Ft7%2Fkr69rlvx5rdd824m61zjqkjr0000gn%2FT%2Fjunit875461810838100905%2Fdataset&lastinstantts=0000003&timelinehash=feda55fdfbaedc993ac653b3410a8951cce50961df61f4b1db9062da7abc95e1
> org.apache.hudi.exception.HoodieMetadataException: Failed to retrieve files
> in partition
> /var/folders/t7/kr69rlvx5rdd824m61zjqkjr0000gn/T/junit875461810838100905/dataset/2015/03/16
> from metadata
> at
> org.apache.hudi.metadata.BaseTableMetadata.getAllFilesInPartition(BaseTableMetadata.java:133)
> at
> org.apache.hudi.metadata.HoodieMetadataFileSystemView.listPartition(HoodieMetadataFileSystemView.java:65)
> at
> org.apache.hudi.common.table.view.AbstractTableFileSystemView.lambda$ensurePartitionLoadedCorrectly$9(AbstractTableFileSystemView.java:304)
> at
> java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1660)
> at
> org.apache.hudi.common.table.view.AbstractTableFileSystemView.ensurePartitionLoadedCorrectly(AbstractTableFileSystemView.java:295)
> at
> org.apache.hudi.common.table.view.AbstractTableFileSystemView.getLatestBaseFilesBeforeOrOn(AbstractTableFileSystemView.java:489)
> at
> org.apache.hudi.timeline.service.handlers.BaseFileHandler.getLatestDataFilesBeforeOrOn(BaseFileHandler.java:60)
> at
> org.apache.hudi.timeline.service.RequestHandler.lambda$registerDataFilesAPI$6(RequestHandler.java:268)
> at
> org.apache.hudi.timeline.service.RequestHandler$ViewHandler.handle(RequestHandler.java:497)
> at io.javalin.security.SecurityUtil.noopAccessManager(SecurityUtil.kt:22)
> at io.javalin.Javalin.lambda$addHandler$0(Javalin.java:606)
> at io.javalin.core.JavalinServlet$service$2$1.invoke(JavalinServlet.kt:46)
> at io.javalin.core.JavalinServlet$service$2$1.invoke(JavalinServlet.kt:17)
> at io.javalin.core.JavalinServlet$service$1.invoke(JavalinServlet.kt:143)
> at io.javalin.core.JavalinServlet$service$2.invoke(JavalinServlet.kt:41)
> at io.javalin.core.JavalinServlet.service(JavalinServlet.kt:107)
> at
> io.javalin.core.util.JettyServerUtil$initialize$httpHandler$1.doHandle(JettyServerUtil.kt:72)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
> at
> org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
> at
> org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1668)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
> at
> org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
> at
> org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:61)
> at
> org.eclipse.jetty.server.handler.StatisticsHandler.handle(StatisticsHandler.java:174)
> at
> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
> at org.eclipse.jetty.server.Server.handle(Server.java:502)
> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:370)
> at
> org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:267)
> at
> org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
> at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)
> at
> org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333)
> at
> org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310)
> at
> org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
> at
> org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126)
> at
> org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366)
> at
> org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
> at
> org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
> at java.lang.Thread.run(Thread.java:748)
> Caused by: org.apache.hudi.exception.HoodieException: Exception when reading
> log file
> at
> org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader.scan(AbstractHoodieLogRecordReader.java:332)
> at
> org.apache.hudi.metadata.HoodieMetadataMergedLogRecordReader.getRecordsByKeys(HoodieMetadataMergedLogRecordReader.java:126)
> at
> org.apache.hudi.metadata.HoodieBackedTableMetadata.readLogRecords(HoodieBackedTableMetadata.java:188)
> at
> org.apache.hudi.metadata.HoodieBackedTableMetadata.lambda$getRecordsByKeys$0(HoodieBackedTableMetadata.java:152)
> at java.util.HashMap.forEach(HashMap.java:1290)
> at
> org.apache.hudi.metadata.HoodieBackedTableMetadata.getRecordsByKeys(HoodieBackedTableMetadata.java:139)
> at
> org.apache.hudi.metadata.HoodieBackedTableMetadata.getRecordByKey(HoodieBackedTableMetadata.java:128)
> at
> org.apache.hudi.metadata.BaseTableMetadata.fetchAllFilesInPartition(BaseTableMetadata.java:285)
> at
> org.apache.hudi.metadata.BaseTableMetadata.getAllFilesInPartition(BaseTableMetadata.java:131)
> ... 39 more
> Caused by: java.lang.NullPointerException
> at
> org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader.processQueuedBlocksForInstant(AbstractHoodieLogRecordReader.java:422)
> at
> org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader.scan(AbstractHoodieLogRecordReader.java:238)
> ... 47 more{noformat}
--
This message was sent by Atlassian Jira
(v8.20.1#820001)
