[
https://issues.apache.org/jira/browse/QPID-6707?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Alex Rudyy updated QPID-6707:
-----------------------------
Status: Reviewable (was: In Progress)
> Kill the Broker on invocation of store mutation operation ending with
> ServerScopedRuntimeException
> --------------------------------------------------------------------------------------------------
>
> Key: QPID-6707
> URL: https://issues.apache.org/jira/browse/QPID-6707
> Project: Qpid
> Issue Type: Bug
> Components: Java Broker
> Reporter: Lorenz Quack
> Assignee: Alex Rudyy
> Fix For: qpid-java-6.0
>
> Attachments:
> 0001-QPID-6707-Java-Broker-Kill-the-Broker-on-invocation-.patch
>
>
> On running out of disk space whilst updating configuration store, thrown
> ServerScopedRuntimeException does not kill the broker.
> Broker continues to run.
> here is example of stack trace:
> {noformat}
> 13:33:14.374 [VirtualHostNode-test-Config] ERROR
> o.a.q.s.u.ServerScopedRuntimeException - Cannot save to store
> java.io.IOException: No space left on device
> at java.io.FileOutputStream.writeBytes(Native Method) ~[na:1.7.0_75]
> at java.io.FileOutputStream.write(FileOutputStream.java:345)
> ~[na:1.7.0_75]
> at
> com.fasterxml.jackson.core.json.UTF8JsonGenerator._flushBuffer(UTF8JsonGenerator.java:1846)
> ~[jackson-core-2.5.0.jar:2.5.0]
> at
> com.fasterxml.jackson.core.json.UTF8JsonGenerator.close(UTF8JsonGenerator.java:1070)
> ~[jackson-core-2.5.0.jar:2.5.0]
> at
> com.fasterxml.jackson.databind.ObjectMapper._configAndWriteValue(ObjectMapper.java:3385)
> ~[jackson-databind-2.5.0.jar:2.5.0]
> at
> com.fasterxml.jackson.databind.ObjectMapper.writeValue(ObjectMapper.java:2726)
> ~[jackson-databind-2.5.0.jar:2.5.0]
> at
> org.apache.qpid.server.store.JsonFileConfigStore$3.performAction(JsonFileConfigStore.java:412)
> ~[classes/:na]
> at
> org.apache.qpid.server.store.JsonFileConfigStore$3.performAction(JsonFileConfigStore.java:408)
> ~[classes/:na]
> at
> org.apache.qpid.server.util.FileHelper.writeFileSafely(FileHelper.java:59)
> ~[classes/:na]
> at
> org.apache.qpid.server.store.JsonFileConfigStore.save(JsonFileConfigStore.java:404)
> [classes/:na]
> at
> org.apache.qpid.server.store.JsonFileConfigStore.update(JsonFileConfigStore.java:572)
> [classes/:na]
> at
> org.apache.qpid.server.configuration.store.StoreConfigurationChangeListener.childAdded(StoreConfigurationChangeListener.java:62)
> [classes/:na]
> at
> org.apache.qpid.server.model.AbstractConfiguredObject.childAdded(AbstractConfiguredObject.java:1435)
> [classes/:na]
> at
> org.apache.qpid.server.model.AbstractConfiguredObject$18$1.call(AbstractConfiguredObject.java:1729)
> [classes/:na]
> at
> org.apache.qpid.server.model.AbstractConfiguredObject$18$1.call(AbstractConfiguredObject.java:1722)
> [classes/:na]
> at
> org.apache.qpid.server.model.AbstractConfiguredObject$21.onSuccess(AbstractConfiguredObject.java:2143)
> [classes/:na]
> at com.google.common.util.concurrent.Futures$6.run(Futures.java:1319)
> [guava-18.0.jar:na]
> at
> org.apache.qpid.server.configuration.updater.TaskExecutorImpl$ImmediateIfSameThreadExecutor.execute(TaskExecutorImpl.java:420)
> [classes/:na]
> at
> com.google.common.util.concurrent.Futures$ImmediateFuture.addListener(Futures.java:106)
> [guava-18.0.jar:na]
> at
> com.google.common.util.concurrent.Futures.addCallback(Futures.java:1322)
> [guava-18.0.jar:na]
> at
> org.apache.qpid.server.model.AbstractConfiguredObject.doAfter(AbstractConfiguredObject.java:2136)
> [classes/:na]
> at
> org.apache.qpid.server.model.AbstractConfiguredObject.doAfter(AbstractConfiguredObject.java:2085)
> [classes/:na]
> at
> org.apache.qpid.server.model.AbstractConfiguredObject$18.call(AbstractConfiguredObject.java:1720)
> [classes/:na]
> at
> org.apache.qpid.server.model.AbstractConfiguredObject$18.call(AbstractConfiguredObject.java:1715)
> [classes/:na]
> at
> org.apache.qpid.server.model.AbstractConfiguredObject$2.execute(AbstractConfiguredObject.java:507)
> [classes/:na]
> at
> org.apache.qpid.server.model.AbstractConfiguredObject$2.execute(AbstractConfiguredObject.java:500)
> [classes/:na]
> at
> org.apache.qpid.server.configuration.updater.TaskExecutorImpl.executeTask(TaskExecutorImpl.java:317)
> [classes/:na]
> at
> org.apache.qpid.server.configuration.updater.TaskExecutorImpl.access$700(TaskExecutorImpl.java:48)
> [classes/:na]
> at
> org.apache.qpid.server.configuration.updater.TaskExecutorImpl$CallableWrapper$1.run(TaskExecutorImpl.java:361)
> [classes/:na]
> at java.security.AccessController.doPrivileged(Native Method)
> [na:1.7.0_75]
> at javax.security.auth.Subject.doAs(Subject.java:356) [na:1.7.0_75]
> at
> org.apache.qpid.server.configuration.updater.TaskExecutorImpl$CallableWrapper.call(TaskExecutorImpl.java:356)
> [classes/:na]
> at java.util.concurrent.FutureTask.run(FutureTask.java:262) [na:1.7.0_75]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> [na:1.7.0_75]
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> [na:1.7.0_75]
> at java.lang.Thread.run(Thread.java:745) [na:1.7.0_75]
> 13:33:14,374 |-ERROR in c.q.l.c.recovery.ResilientFileOutputStream@1367869763
> - IO failure while writing to file [/mnt/vfs/work/log/qpid.log]
> java.io.IOException: No space left on device
> at java.io.IOException: No space left on device
> at at java.io.FileOutputStream.writeBytes(Native Method)
> at at java.io.FileOutputStream.write(FileOutputStream.java:345)
> at at
> java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
> at at
> java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
> at at
> ch.qos.logback.core.recovery.ResilientOutputStreamBase.flush(ResilientOutputStreamBase.java:79)
> at at
> ch.qos.logback.core.encoder.LayoutWrappingEncoder.doEncode(LayoutWrappingEncoder.java:137)
> at at
> ch.qos.logback.core.OutputStreamAppender.writeOut(OutputStreamAppender.java:194)
> at at ch.qos.logback.core.FileAppender.writeOut(FileAppender.java:209)
> at at
> ch.qos.logback.core.OutputStreamAppender.subAppend(OutputStreamAppender.java:219)
> at at
> ch.qos.logback.core.rolling.RollingFileAppender.subAppend(RollingFileAppender.java:182)
> at at
> ch.qos.logback.core.OutputStreamAppender.append(OutputStreamAppender.java:103)
> at at
> ch.qos.logback.core.UnsynchronizedAppenderBase.doAppend(UnsynchronizedAppenderBase.java:88)
> at at
> ch.qos.logback.core.spi.AppenderAttachableImpl.appendLoopOnAppenders(AppenderAttachableImpl.java:48)
> at at
> ch.qos.logback.classic.Logger.appendLoopOnAppenders(Logger.java:273)
> at at ch.qos.logback.classic.Logger.callAppenders(Logger.java:260)
> at at
> ch.qos.logback.classic.Logger.buildLoggingEventAndAppend(Logger.java:442)
> at at
> ch.qos.logback.classic.Logger.filterAndLog_0_Or3Plus(Logger.java:396)
> at at ch.qos.logback.classic.Logger.error(Logger.java:559)
> at at
> org.apache.qpid.server.util.ServerScopedRuntimeException.<init>(ServerScopedRuntimeException.java:40)
> at at
> org.apache.qpid.server.store.StoreException.<init>(StoreException.java:34)
> at at
> org.apache.qpid.server.store.JsonFileConfigStore.save(JsonFileConfigStore.java:418)
> at at
> org.apache.qpid.server.store.JsonFileConfigStore.update(JsonFileConfigStore.java:572)
> at at
> org.apache.qpid.server.configuration.store.StoreConfigurationChangeListener.childAdded(StoreConfigurationChangeListener.java:62)
> at at
> org.apache.qpid.server.model.AbstractConfiguredObject.childAdded(AbstractConfiguredObject.java:1435)
> at at
> org.apache.qpid.server.model.AbstractConfiguredObject$18$1.call(AbstractConfiguredObject.java:1729)
> at at
> org.apache.qpid.server.model.AbstractConfiguredObject$18$1.call(AbstractConfiguredObject.java:1722)
> at at
> org.apache.qpid.server.model.AbstractConfiguredObject$21.onSuccess(AbstractConfiguredObject.java:2143)
> at at
> com.google.common.util.concurrent.Futures$6.run(Futures.java:1319)
> at at
> org.apache.qpid.server.configuration.updater.TaskExecutorImpl$ImmediateIfSameThreadExecutor.execute(TaskExecutorImpl.java:420)
> at at
> com.google.common.util.concurrent.Futures$ImmediateFuture.addListener(Futures.java:106)
> at at
> com.google.common.util.concurrent.Futures.addCallback(Futures.java:1322)
> at at
> org.apache.qpid.server.model.AbstractConfiguredObject.doAfter(AbstractConfiguredObject.java:2136)
> at at
> org.apache.qpid.server.model.AbstractConfiguredObject.doAfter(AbstractConfiguredObject.java:2085)
> at at
> org.apache.qpid.server.model.AbstractConfiguredObject$18.call(AbstractConfiguredObject.java:1720)
> at at
> org.apache.qpid.server.model.AbstractConfiguredObject$18.call(AbstractConfiguredObject.java:1715)
> at at
> org.apache.qpid.server.model.AbstractConfiguredObject$2.execute(AbstractConfiguredObject.java:507)
> at at
> org.apache.qpid.server.model.AbstractConfiguredObject$2.execute(AbstractConfiguredObject.java:500)
> at at
> org.apache.qpid.server.configuration.updater.TaskExecutorImpl.executeTask(TaskExecutorImpl.java:317)
> at at
> org.apache.qpid.server.configuration.updater.TaskExecutorImpl.access$700(TaskExecutorImpl.java:48)
> at at
> org.apache.qpid.server.configuration.updater.TaskExecutorImpl$CallableWrapper$1.run(TaskExecutorImpl.java:361)
> at at java.security.AccessController.doPrivileged(Native Method)
> at at javax.security.auth.Subject.doAs(Subject.java:356)
> at at
> org.apache.qpid.server.configuration.updater.TaskExecutorImpl$CallableWrapper.call(TaskExecutorImpl.java:356)
> at at java.util.concurrent.FutureTask.run(FutureTask.java:262)
> at at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> at at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> at at java.lang.Thread.run(Thread.java:745)
> 13:33:14.375 [VirtualHostNode-test-Config] ERROR
> o.a.q.s.logging.BrokerFileLoggerImpl - Unexpected I/O error whilst trying to
> write to log file. Log messages could be lost.
> java.io.IOException: No space left on device
> at java.io.FileOutputStream.writeBytes(Native Method) ~[na:1.7.0_75]
> at java.io.FileOutputStream.write(FileOutputStream.java:345)
> ~[na:1.7.0_75]
> at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
> ~[na:1.7.0_75]
> at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
> ~[na:1.7.0_75]
> at
> ch.qos.logback.core.recovery.ResilientOutputStreamBase.flush(ResilientOutputStreamBase.java:79)
> ~[logback-core-1.1.2.jar:na]
> at
> ch.qos.logback.core.encoder.LayoutWrappingEncoder.doEncode(LayoutWrappingEncoder.java:137)
> [logback-core-1.1.2.jar:na]
> at
> ch.qos.logback.core.OutputStreamAppender.writeOut(OutputStreamAppender.java:194)
> [logback-core-1.1.2.jar:na]
> at ch.qos.logback.core.FileAppender.writeOut(FileAppender.java:209)
> [logback-core-1.1.2.jar:na]
> at
> ch.qos.logback.core.OutputStreamAppender.subAppend(OutputStreamAppender.java:219)
> [logback-core-1.1.2.jar:na]
> at
> ch.qos.logback.core.rolling.RollingFileAppender.subAppend(RollingFileAppender.java:182)
> [logback-core-1.1.2.jar:na]
> at
> ch.qos.logback.core.OutputStreamAppender.append(OutputStreamAppender.java:103)
> [logback-core-1.1.2.jar:na]
> at
> ch.qos.logback.core.UnsynchronizedAppenderBase.doAppend(UnsynchronizedAppenderBase.java:88)
> [logback-core-1.1.2.jar:na]
> at
> ch.qos.logback.core.spi.AppenderAttachableImpl.appendLoopOnAppenders(AppenderAttachableImpl.java:48)
> [logback-core-1.1.2.jar:na]
> at ch.qos.logback.classic.Logger.appendLoopOnAppenders(Logger.java:273)
> [logback-classic-1.1.2.jar:na]
> at ch.qos.logback.classic.Logger.callAppenders(Logger.java:260)
> [logback-classic-1.1.2.jar:na]
> at
> ch.qos.logback.classic.Logger.buildLoggingEventAndAppend(Logger.java:442)
> [logback-classic-1.1.2.jar:na]
> at ch.qos.logback.classic.Logger.filterAndLog_0_Or3Plus(Logger.java:396)
> [logback-classic-1.1.2.jar:na]
> at ch.qos.logback.classic.Logger.error(Logger.java:559)
> [logback-classic-1.1.2.jar:na]
> at
> org.apache.qpid.server.util.ServerScopedRuntimeException.<init>(ServerScopedRuntimeException.java:40)
> [classes/:na]
> at
> org.apache.qpid.server.store.StoreException.<init>(StoreException.java:34)
> [classes/:na]
> at
> org.apache.qpid.server.store.JsonFileConfigStore.save(JsonFileConfigStore.java:418)
> [classes/:na]
> at
> org.apache.qpid.server.store.JsonFileConfigStore.update(JsonFileConfigStore.java:572)
> [classes/:na]
> at
> org.apache.qpid.server.configuration.store.StoreConfigurationChangeListener.childAdded(StoreConfigurationChangeListener.java:62)
> [classes/:na]
> at
> org.apache.qpid.server.model.AbstractConfiguredObject.childAdded(AbstractConfiguredObject.java:1435)
> [classes/:na]
> at
> org.apache.qpid.server.model.AbstractConfiguredObject$18$1.call(AbstractConfiguredObject.java:1729)
> [classes/:na]
> at
> org.apache.qpid.server.model.AbstractConfiguredObject$18$1.call(AbstractConfiguredObject.java:1722)
> [classes/:na]
> at
> org.apache.qpid.server.model.AbstractConfiguredObject$21.onSuccess(AbstractConfiguredObject.java:2143)
> [classes/:na]
> at com.google.common.util.concurrent.Futures$6.run(Futures.java:1319)
> [guava-18.0.jar:na]
> at
> org.apache.qpid.server.configuration.updater.TaskExecutorImpl$ImmediateIfSameThreadExecutor.execute(TaskExecutorImpl.java:420)
> [classes/:na]
> at
> com.google.common.util.concurrent.Futures$ImmediateFuture.addListener(Futures.java:106)
> [guava-18.0.jar:na]
> at
> com.google.common.util.concurrent.Futures.addCallback(Futures.java:1322)
> [guava-18.0.jar:na]
> at
> org.apache.qpid.server.model.AbstractConfiguredObject.doAfter(AbstractConfiguredObject.java:2136)
> [classes/:na]
> at
> org.apache.qpid.server.model.AbstractConfiguredObject.doAfter(AbstractConfiguredObject.java:2085)
> [classes/:na]
> at
> org.apache.qpid.server.model.AbstractConfiguredObject$18.call(AbstractConfiguredObject.java:1720)
> [classes/:na]
> at
> org.apache.qpid.server.model.AbstractConfiguredObject$18.call(AbstractConfiguredObject.java:1715)
> [classes/:na]
> at
> org.apache.qpid.server.model.AbstractConfiguredObject$2.execute(AbstractConfiguredObject.java:507)
> [classes/:na]
> at
> org.apache.qpid.server.model.AbstractConfiguredObject$2.execute(AbstractConfiguredObject.java:500)
> [classes/:na]
> at
> org.apache.qpid.server.configuration.updater.TaskExecutorImpl.executeTask(TaskExecutorImpl.java:317)
> [classes/:na]
> at
> org.apache.qpid.server.configuration.updater.TaskExecutorImpl.access$700(TaskExecutorImpl.java:48)
> [classes/:na]
> at
> org.apache.qpid.server.configuration.updater.TaskExecutorImpl$CallableWrapper$1.run(TaskExecutorImpl.java:361)
> [classes/:na]
> at java.security.AccessController.doPrivileged(Native Method)
> [na:1.7.0_75]
> at javax.security.auth.Subject.doAs(Subject.java:356) [na:1.7.0_75]
> at
> org.apache.qpid.server.configuration.updater.TaskExecutorImpl$CallableWrapper.call(TaskExecutorImpl.java:356)
> [classes/:na]
> at java.util.concurrent.FutureTask.run(FutureTask.java:262) [na:1.7.0_75]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> [na:1.7.0_75]
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> [na:1.7.0_75]
> at java.lang.Thread.run(Thread.java:745) [na:1.7.0_75]
> 13:33:14.377 [HttpManagement-58] WARN o.a.q.s.m.p.servlet.rest.RestServlet -
> Unexpected exception processing request
> org.apache.qpid.server.store.StoreException: Cannot save to store
> at
> org.apache.qpid.server.store.JsonFileConfigStore.save(JsonFileConfigStore.java:418)
> ~[classes/:na]
> at
> org.apache.qpid.server.store.JsonFileConfigStore.update(JsonFileConfigStore.java:572)
> ~[classes/:na]
> at
> org.apache.qpid.server.configuration.store.StoreConfigurationChangeListener.childAdded(StoreConfigurationChangeListener.java:62)
> ~[classes/:na]
> at
> org.apache.qpid.server.model.AbstractConfiguredObject.childAdded(AbstractConfiguredObject.java:1435)
> ~[classes/:na]
> at
> org.apache.qpid.server.model.AbstractConfiguredObject$18$1.call(AbstractConfiguredObject.java:1729)
> ~[classes/:na]
> at
> org.apache.qpid.server.model.AbstractConfiguredObject$18$1.call(AbstractConfiguredObject.java:1722)
> ~[classes/:na]
> at
> org.apache.qpid.server.model.AbstractConfiguredObject$21.onSuccess(AbstractConfiguredObject.java:2143)
> ~[classes/:na]
> at com.google.common.util.concurrent.Futures$6.run(Futures.java:1319)
> ~[guava-18.0.jar:na]
> at
> org.apache.qpid.server.configuration.updater.TaskExecutorImpl$ImmediateIfSameThreadExecutor.execute(TaskExecutorImpl.java:420)
> ~[classes/:na]
> at
> com.google.common.util.concurrent.Futures$ImmediateFuture.addListener(Futures.java:106)
> ~[guava-18.0.jar:na]
> at
> com.google.common.util.concurrent.Futures.addCallback(Futures.java:1322)
> ~[guava-18.0.jar:na]
> at
> org.apache.qpid.server.model.AbstractConfiguredObject.doAfter(AbstractConfiguredObject.java:2136)
> ~[classes/:na]
> at
> org.apache.qpid.server.model.AbstractConfiguredObject.doAfter(AbstractConfiguredObject.java:2085)
> ~[classes/:na]
> at
> org.apache.qpid.server.model.AbstractConfiguredObject$18.call(AbstractConfiguredObject.java:1720)
> ~[classes/:na]
> at
> org.apache.qpid.server.model.AbstractConfiguredObject$18.call(AbstractConfiguredObject.java:1715)
> ~[classes/:na]
> at
> org.apache.qpid.server.model.AbstractConfiguredObject$2.execute(AbstractConfiguredObject.java:507)
> ~[classes/:na]
> at
> org.apache.qpid.server.model.AbstractConfiguredObject$2.execute(AbstractConfiguredObject.java:500)
> ~[classes/:na]
> at
> org.apache.qpid.server.configuration.updater.TaskExecutorImpl.executeTask(TaskExecutorImpl.java:317)
> ~[classes/:na]
> at
> org.apache.qpid.server.configuration.updater.TaskExecutorImpl.access$700(TaskExecutorImpl.java:48)
> ~[classes/:na]
> at
> org.apache.qpid.server.configuration.updater.TaskExecutorImpl$CallableWrapper$1.run(TaskExecutorImpl.java:361)
> ~[classes/:na]
> at java.security.AccessController.doPrivileged(Native Method)
> ~[na:1.7.0_75]
> at javax.security.auth.Subject.doAs(Subject.java:356) ~[na:1.7.0_75]
> at
> org.apache.qpid.server.configuration.updater.TaskExecutorImpl$CallableWrapper.call(TaskExecutorImpl.java:356)
> ~[classes/:na]
> at java.util.concurrent.FutureTask.run(FutureTask.java:262) ~[na:1.7.0_75]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> ~[na:1.7.0_75]
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> ~[na:1.7.0_75]
> at java.lang.Thread.run(Thread.java:745) [na:1.7.0_75]
> Caused by: java.io.IOException: No space left on device
> at java.io.FileOutputStream.writeBytes(Native Method) ~[na:1.7.0_75]
> at java.io.FileOutputStream.write(FileOutputStream.java:345)
> ~[na:1.7.0_75]
> at
> com.fasterxml.jackson.core.json.UTF8JsonGenerator._flushBuffer(UTF8JsonGenerator.java:1846)
> ~[jackson-core-2.5.0.jar:2.5.0]
> at
> com.fasterxml.jackson.core.json.UTF8JsonGenerator.close(UTF8JsonGenerator.java:1070)
> ~[jackson-core-2.5.0.jar:2.5.0]
> at
> com.fasterxml.jackson.databind.ObjectMapper._configAndWriteValue(ObjectMapper.java:3385)
> ~[jackson-databind-2.5.0.jar:2.5.0]
> at
> com.fasterxml.jackson.databind.ObjectMapper.writeValue(ObjectMapper.java:2726)
> ~[jackson-databind-2.5.0.jar:2.5.0]
> at
> org.apache.qpid.server.store.JsonFileConfigStore$3.performAction(JsonFileConfigStore.java:412)
> ~[classes/:na]
> at
> org.apache.qpid.server.store.JsonFileConfigStore$3.performAction(JsonFileConfigStore.java:408)
> ~[classes/:na]
> at
> org.apache.qpid.server.util.FileHelper.writeFileSafely(FileHelper.java:59)
> ~[classes/:na]
> at
> org.apache.qpid.server.store.JsonFileConfigStore.save(JsonFileConfigStore.java:404)
> ~[classes/:na]
> ... 26 common frames omitted
> {noformat}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
