[
https://issues.apache.org/jira/browse/NIFI-902?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14721150#comment-14721150
]
Mark Payne commented on NIFI-902:
---------------------------------
[~bende]: While I believe that the supplied patch addressed the problem as it
was stated in the ticket, I was testing more thoroughly and ran into another
issue that occurred when the FlowFile repo ran out of disk space. The repo
ended up becoming corrupt and when I tried to restart NiFi, I got the following
stack trace:
{noformat}
2015-08-28 16:13:35,486 WARN [main] org.apache.nifi.web.server.JettyServer
Failed to start web server... shutting down.
org.apache.nifi.web.NiFiCoreException: Unable to start Flow Controller.
at
org.apache.nifi.web.contextlistener.ApplicationStartupContextListener.contextInitialized(ApplicationStartupContextListener.java:99)
~[na:na]
at
org.eclipse.jetty.server.handler.ContextHandler.callContextInitialized(ContextHandler.java:800)
~[jetty-server-9.2.11.v20150529.jar:9.2.11.v20150529]
at
org.eclipse.jetty.servlet.ServletContextHandler.callContextInitialized(ServletContextHandler.java:444)
~[jetty-servlet-9.2.11.v20150529.jar:9.2.11.v20150529]
at
org.eclipse.jetty.server.handler.ContextHandler.startContext(ContextHandler.java:791)
~[jetty-server-9.2.11.v20150529.jar:9.2.11.v20150529]
at
org.eclipse.jetty.servlet.ServletContextHandler.startContext(ServletContextHandler.java:294)
~[jetty-servlet-9.2.11.v20150529.jar:9.2.11.v20150529]
at
org.eclipse.jetty.webapp.WebAppContext.startWebapp(WebAppContext.java:1349)
~[jetty-webapp-9.2.11.v20150529.jar:9.2.11.v20150529]
at
org.eclipse.jetty.webapp.WebAppContext.startContext(WebAppContext.java:1342)
~[jetty-webapp-9.2.11.v20150529.jar:9.2.11.v20150529]
at
org.eclipse.jetty.server.handler.ContextHandler.doStart(ContextHandler.java:741)
~[jetty-server-9.2.11.v20150529.jar:9.2.11.v20150529]
at
org.eclipse.jetty.webapp.WebAppContext.doStart(WebAppContext.java:505)
~[jetty-webapp-9.2.11.v20150529.jar:9.2.11.v20150529]
at
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
~[jetty-util-9.2.11.v20150529.jar:9.2.11.v20150529]
at
org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:132)
~[jetty-util-9.2.11.v20150529.jar:9.2.11.v20150529]
at
org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:114)
~[jetty-util-9.2.11.v20150529.jar:9.2.11.v20150529]
at
org.eclipse.jetty.server.handler.AbstractHandler.doStart(AbstractHandler.java:61)
~[jetty-server-9.2.11.v20150529.jar:9.2.11.v20150529]
at
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
~[jetty-util-9.2.11.v20150529.jar:9.2.11.v20150529]
at
org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:132)
~[jetty-util-9.2.11.v20150529.jar:9.2.11.v20150529]
at org.eclipse.jetty.server.Server.start(Server.java:387)
~[jetty-server-9.2.11.v20150529.jar:9.2.11.v20150529]
at
org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:114)
~[jetty-util-9.2.11.v20150529.jar:9.2.11.v20150529]
at
org.eclipse.jetty.server.handler.AbstractHandler.doStart(AbstractHandler.java:61)
~[jetty-server-9.2.11.v20150529.jar:9.2.11.v20150529]
at org.eclipse.jetty.server.Server.doStart(Server.java:354)
~[jetty-server-9.2.11.v20150529.jar:9.2.11.v20150529]
at
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
~[jetty-util-9.2.11.v20150529.jar:9.2.11.v20150529]
at org.apache.nifi.web.server.JettyServer.start(JettyServer.java:658)
~[nifi-jetty-0.3.0-SNAPSHOT.jar:0.3.0-SNAPSHOT]
at org.apache.nifi.NiFi.<init>(NiFi.java:137)
[nifi-runtime-0.3.0-SNAPSHOT.jar:0.3.0-SNAPSHOT]
at org.apache.nifi.NiFi.main(NiFi.java:227)
[nifi-runtime-0.3.0-SNAPSHOT.jar:0.3.0-SNAPSHOT]
Caused by: java.io.IOException: Claim Existence Qualifier not found in stream;
found value: 224 after successfully restoring 111070 records
at
org.apache.nifi.controller.repository.WriteAheadFlowFileRepository$WriteAheadRecordSerde.deserializeClaim(WriteAheadFlowFileRepository.java:787)
~[nifi-framework-core-0.3.0-SNAPSHOT.jar:0.3.0-SNAPSHOT]
at
org.apache.nifi.controller.repository.WriteAheadFlowFileRepository$WriteAheadRecordSerde.deserializeEdit(WriteAheadFlowFileRepository.java:572)
~[nifi-framework-core-0.3.0-SNAPSHOT.jar:0.3.0-SNAPSHOT]
at
org.apache.nifi.controller.repository.WriteAheadFlowFileRepository$WriteAheadRecordSerde.deserializeEdit(WriteAheadFlowFileRepository.java:395)
~[nifi-framework-core-0.3.0-SNAPSHOT.jar:0.3.0-SNAPSHOT]
at
org.wali.MinimalLockingWriteAheadLog$Partition.recoverNextTransaction(MinimalLockingWriteAheadLog.java:977)
~[nifi-write-ahead-log-0.3.0-SNAPSHOT.jar:0.3.0-SNAPSHOT]
at
org.wali.MinimalLockingWriteAheadLog.recoverFromEdits(MinimalLockingWriteAheadLog.java:447)
~[nifi-write-ahead-log-0.3.0-SNAPSHOT.jar:0.3.0-SNAPSHOT]
at
org.wali.MinimalLockingWriteAheadLog.recoverRecords(MinimalLockingWriteAheadLog.java:292)
~[nifi-write-ahead-log-0.3.0-SNAPSHOT.jar:0.3.0-SNAPSHOT]
at
org.apache.nifi.controller.repository.WriteAheadFlowFileRepository.loadFlowFiles(WriteAheadFlowFileRepository.java:328)
~[nifi-framework-core-0.3.0-SNAPSHOT.jar:0.3.0-SNAPSHOT]
at
org.apache.nifi.controller.FlowController.initializeFlow(FlowController.java:557)
~[nifi-framework-core-0.3.0-SNAPSHOT.jar:0.3.0-SNAPSHOT]
at
org.apache.nifi.controller.StandardFlowService.loadFromBytes(StandardFlowService.java:622)
~[nifi-framework-core-0.3.0-SNAPSHOT.jar:0.3.0-SNAPSHOT]
at
org.apache.nifi.controller.StandardFlowService.load(StandardFlowService.java:458)
~[nifi-framework-core-0.3.0-SNAPSHOT.jar:0.3.0-SNAPSHOT]
at
org.apache.nifi.web.contextlistener.ApplicationStartupContextListener.contextInitialized(ApplicationStartupContextListener.java:79)
~[na:na]
... 22 common frames omitted
{noformat}
Rather than create an additional ticket that indicates that we don't handle
failure of writing to the FlowFile repo well, I decided to just change the
title of this ticket to make it a little more broad. I believe that I
understand the reason for the failure, and I will post another patch shortly.
> NiFi does not properly handle error conditions if writing to FlowFile
> repository fails
> --------------------------------------------------------------------------------------
>
> Key: NIFI-902
> URL: https://issues.apache.org/jira/browse/NIFI-902
> Project: Apache NiFi
> Issue Type: Bug
> Components: Core Framework
> Reporter: Mark Payne
> Assignee: Mark Payne
> Priority: Critical
> Fix For: 0.3.0
>
> Attachments:
> 0001-NIFI-902-Ensure-that-if-FlowFile-Repo-runs-out-of-di.patch
>
>
> I ran out of disk space on my FlowFile Repository. After cleaning up some
> files on the partition, I was able to continue running but when the FlowFile
> Repo attempted to checkpoint, I got the following Exception in the logs:
> {noformat}
> 2015-08-27 15:59:58,398 ERROR [pool-20-thread-1]
> o.a.n.c.r.WriteAheadFlowFileRepository Unable to checkpoint FlowFile
> Repository due to java.io.IOException: Stream Closed
> java.io.IOException: Stream Closed
> at java.io.FileOutputStream.writeBytes(Native Method) ~[na:1.8.0_45]
> at java.io.FileOutputStream.write(FileOutputStream.java:326)
> ~[na:1.8.0_45]
> at
> org.apache.nifi.stream.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:69)
> ~[nifi-utils-0.3.0-SNAPSHOT.jar:0.3.0-SNAPSHOT]
> at
> org.apache.nifi.stream.io.BufferedOutputStream.flush(BufferedOutputStream.java:126)
> ~[nifi-utils-0.3.0-SNAPSHOT.jar:0.3.0-SNAPSHOT]
> at java.io.DataOutputStream.flush(DataOutputStream.java:123)
> ~[na:1.8.0_45]
> at java.io.FilterOutputStream.close(FilterOutputStream.java:158)
> ~[na:1.8.0_45]
> at
> org.wali.MinimalLockingWriteAheadLog$Partition.rollover(MinimalLockingWriteAheadLog.java:723)
> ~[nifi-write-ahead-log-0.3.0-SNAPSHOT.jar:0.3.0-SNAPSHOT]
> at
> org.wali.MinimalLockingWriteAheadLog.checkpoint(MinimalLockingWriteAheadLog.java:512)
> ~[nifi-write-ahead-log-0.3.0-SNAPSHOT.jar:0.3.0-SNAPSHOT]
> at
> org.apache.nifi.controller.repository.WriteAheadFlowFileRepository.checkpoint(WriteAheadFlowFileRepository.java:392)
> ~[nifi-framework-core-0.3.0-SNAPSHOT.jar:0.3.0-SNAPSHOT]
> at
> org.apache.nifi.controller.repository.WriteAheadFlowFileRepository$1.run(WriteAheadFlowFileRepository.java:364)
> ~[nifi-framework-core-0.3.0-SNAPSHOT.jar:0.3.0-SNAPSHOT]
> at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
> [na:1.8.0_45]
> at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
> [na:1.8.0_45]
> at
> java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
> [na:1.8.0_45]
> at
> java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
> [na:1.8.0_45]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> [na:1.8.0_45]
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> [na:1.8.0_45]
> at java.lang.Thread.run(Thread.java:745) [na:1.8.0_45]
> Suppressed: java.io.IOException: Stream Closed
> at java.io.FileOutputStream.writeBytes(Native Method)
> ~[na:1.8.0_45]
> at java.io.FileOutputStream.write(FileOutputStream.java:326)
> ~[na:1.8.0_45]
> at
> org.apache.nifi.stream.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:69)
> ~[nifi-utils-0.3.0-SNAPSHOT.jar:0.3.0-SNAPSHOT]
> at
> org.apache.nifi.stream.io.BufferedOutputStream.flush(BufferedOutputStream.java:126)
> ~[nifi-utils-0.3.0-SNAPSHOT.jar:0.3.0-SNAPSHOT]
> at
> java.io.FilterOutputStream.close(FilterOutputStream.java:158) ~[na:1.8.0_45]
> at
> java.io.FilterOutputStream.close(FilterOutputStream.java:159) ~[na:1.8.0_45]
> ... 11 common frames omitted
> {noformat}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
