Hi all, We're encountering an issue where a longer API request ends up aborting just after 60s (the DB query itself takes approximately 60s) and returning a 504 (Gateway Timeout). The exception thrown in the logs is attached. It looks like the socket is suddenly being closed; I suspect it has something to do with either Tomcat or JDBC timeouts, but all of my attempts to set those to longer values (connectionTimeout, keepAliveTimeout in server.xml, removeAbandonedTimeout in the JDBC connection) have not had any affect.
The timeouts do not seem to be triggered client-side (they occur even if we set an indefinite timeout, or even just using curl which has a longer default timeout). Has anyone run into this before with long-running queries or API calls and found a solution? Best, Arvind
Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: 03:38:31.202 [http-bio-8443-exec-57] ERROR o.s.boot.context.web.ErrorPageFilter - Forwarding to error page from request [/api/v1/loans] due to exception [null] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: org.apache.catalina.connector.ClientAbortException: null Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.connector.OutputBuffer.realWriteBytes(OutputBuffer.java:406) ~[catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.tomcat.util.buf.ByteChunk.append(ByteChunk.java:342) ~[tomcat-coyote.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.connector.OutputBuffer.writeBytes(OutputBuffer.java:431) ~[catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.connector.OutputBuffer.write(OutputBuffer.java:419) ~[catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.connector.CoyoteOutputStream.write(CoyoteOutputStream.java:91) ~[catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at com.sun.jersey.spi.container.servlet.WebComponent$Writer.write(WebComponent.java:307) ~[jersey-servlet-1.17.jar:1.17] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at com.sun.jersey.spi.container.ContainerResponse$CommittingOutputStream.write(ContainerResponse.java:134) ~[jersey-server-1.17.jar:1.17] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221) ~[na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at sun.nio.cs.StreamEncoder.implWrite(StreamEncoder.java:282) ~[na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:125) ~[na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at java.io.OutputStreamWriter.write(OutputStreamWriter.java:207) ~[na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at java.io.BufferedWriter.flushBuffer(BufferedWriter.java:129) ~[na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at java.io.BufferedWriter.write(BufferedWriter.java:230) ~[na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at java.io.Writer.write(Writer.java:157) ~[na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at com.sun.jersey.core.util.ReaderWriter.writeToAsString(ReaderWriter.java:190) ~[jersey-core-1.17.jar:1.17] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at com.sun.jersey.core.provider.AbstractMessageReaderWriterProvider.writeToAsString(AbstractMessageReaderWriterProvider.java:128) ~[jersey-core-1.17.jar:1.17] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at com.sun.jersey.core.impl.provider.entity.StringProvider.writeTo(StringProvider.java:88) ~[jersey-core-1.17.jar:1.17] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at com.sun.jersey.core.impl.provider.entity.StringProvider.writeTo(StringProvider.java:58) ~[jersey-core-1.17.jar:1.17] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at com.sun.jersey.spi.container.ContainerResponse.write(ContainerResponse.java:306) ~[jersey-server-1.17.jar:1.17] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1479) ~[jersey-server-1.17.jar:1.17] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1391) ~[jersey-server-1.17.jar:1.17] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1381) ~[jersey-server-1.17.jar:1.17] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:416) ~[jersey-servlet-1.17.jar:1.17] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:538) ~[jersey-servlet-1.17.jar:1.17] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:716) ~[jersey-servlet-1.17.jar:1.17] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at javax.servlet.http.HttpServlet.service(HttpServlet.java:728) ~[servlet-api.jar:na] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:305) [catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210) [catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.access.intercept.FilterSecurityInterceptor.invoke(FilterSecurityInterceptor.java:108) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.access.intercept.FilterSecurityInterceptor.doFilter(FilterSecurityInterceptor.java:84) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:243) [catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210) [catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:330) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.access.intercept.FilterSecurityInterceptor.invoke(FilterSecurityInterceptor.java:118) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.access.intercept.FilterSecurityInterceptor.doFilter(FilterSecurityInterceptor.java:84) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.access.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:113) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.authentication.AnonymousAuthenticationFilter.doFilter(AnonymousAuthenticationFilter.java:113) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.servletapi.SecurityContextHolderAwareRequestFilter.doFilter(SecurityContextHolderAwareRequestFilter.java:154) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.context.request.async.WebAsyncManagerIntegrationFilter.doFilterInternal(WebAsyncManagerIntegrationFilter.java:50) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107) [spring-web-4.0.8.RELEASE.jar:4.0.8.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.authentication.www.BasicAuthenticationFilter.doFilter(BasicAuthenticationFilter.java:201) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.fineract.infrastructure.security.filter.TenantAwareBasicAuthenticationFilter.doFilter(TenantAwareBasicAuthenticationFilter.java:150) ~[TenantAwareBasicAuthenticationFilter.class:na] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.context.SecurityContextPersistenceFilter.doFilter(SecurityContextPersistenceFilter.java:87) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.access.channel.ChannelProcessingFilter.doFilter(ChannelProcessingFilter.java:144) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.FilterChainProxy.doFilterInternal(FilterChainProxy.java:192) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.FilterChainProxy.doFilter(FilterChainProxy.java:160) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:243) [catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210) [catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.boot.context.web.ErrorPageFilter.doFilter(ErrorPageFilter.java:108) [spring-boot-1.1.6.RELEASE.jar:1.1.6.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.boot.context.web.ErrorPageFilter.access$000(ErrorPageFilter.java:58) [spring-boot-1.1.6.RELEASE.jar:1.1.6.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.boot.context.web.ErrorPageFilter$1.doFilterInternal(ErrorPageFilter.java:87) [spring-boot-1.1.6.RELEASE.jar:1.1.6.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107) [spring-web-4.0.8.RELEASE.jar:4.0.8.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.boot.context.web.ErrorPageFilter.doFilter(ErrorPageFilter.java:100) [spring-boot-1.1.6.RELEASE.jar:1.1.6.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:243) [catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210) [catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:222) [catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:123) [catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:502) [catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:171) [catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:99) [catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:953) [catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:118) [catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:408) [catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1023) [tomcat-coyote.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:589) [tomcat-coyote.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:312) [tomcat-coyote.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at java.lang.Thread.run(Thread.java:745) [na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: Caused by: java.net.SocketException: Broken pipe Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at java.net.SocketOutputStream.socketWrite0(Native Method) ~[na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:109) ~[na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at java.net.SocketOutputStream.write(SocketOutputStream.java:153) ~[na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at sun.security.ssl.OutputRecord.writeBuffer(OutputRecord.java:431) ~[na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at sun.security.ssl.OutputRecord.write(OutputRecord.java:417) ~[na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at sun.security.ssl.SSLSocketImpl.writeRecordInternal(SSLSocketImpl.java:864) ~[na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at sun.security.ssl.SSLSocketImpl.writeRecord(SSLSocketImpl.java:835) ~[na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at sun.security.ssl.AppOutputStream.write(AppOutputStream.java:123) ~[na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.coyote.http11.InternalOutputBuffer.realWriteBytes(InternalOutputBuffer.java:215) ~[tomcat-coyote.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.tomcat.util.buf.ByteChunk.flushBuffer(ByteChunk.java:480) ~[tomcat-coyote.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.tomcat.util.buf.ByteChunk.append(ByteChunk.java:366) ~[tomcat-coyote.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.coyote.http11.InternalOutputBuffer$OutputStreamOutputBuffer.doWrite(InternalOutputBuffer.java:240) ~[tomcat-coyote.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.coyote.http11.filters.ChunkedOutputFilter.doWrite(ChunkedOutputFilter.java:119) ~[tomcat-coyote.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.coyote.http11.AbstractOutputBuffer.doWrite(AbstractOutputBuffer.java:192) ~[tomcat-coyote.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.coyote.Response.doWrite(Response.java:517) ~[tomcat-coyote.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.connector.OutputBuffer.realWriteBytes(OutputBuffer.java:401) ~[catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011... 76 common frames omitted Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: 03:38:31.204 [http-bio-8443-exec-57] ERROR o.s.boot.context.web.ErrorPageFilter - Cannot forward to error page for/fineract-provider/api/v1/loans (response is committed), so this response may have the wrong status code Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: org.apache.catalina.connector.ClientAbortException: null Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.connector.OutputBuffer.realWriteBytes(OutputBuffer.java:406) ~[catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.tomcat.util.buf.ByteChunk.append(ByteChunk.java:342) ~[tomcat-coyote.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.connector.OutputBuffer.writeBytes(OutputBuffer.java:431) ~[catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.connector.OutputBuffer.write(OutputBuffer.java:419) ~[catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.connector.CoyoteOutputStream.write(CoyoteOutputStream.java:91) ~[catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at com.sun.jersey.spi.container.servlet.WebComponent$Writer.write(WebComponent.java:307) ~[jersey-servlet-1.17.jar:1.17] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at com.sun.jersey.spi.container.ContainerResponse$CommittingOutputStream.write(ContainerResponse.java:134) ~[jersey-server-1.17.jar:1.17] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221) ~[na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at sun.nio.cs.StreamEncoder.implWrite(StreamEncoder.java:282) ~[na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:125) ~[na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at java.io.OutputStreamWriter.write(OutputStreamWriter.java:207) ~[na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at java.io.BufferedWriter.flushBuffer(BufferedWriter.java:129) ~[na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at java.io.BufferedWriter.write(BufferedWriter.java:230) ~[na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at java.io.Writer.write(Writer.java:157) ~[na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at com.sun.jersey.core.util.ReaderWriter.writeToAsString(ReaderWriter.java:190) ~[jersey-core-1.17.jar:1.17] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at com.sun.jersey.core.provider.AbstractMessageReaderWriterProvider.writeToAsString(AbstractMessageReaderWriterProvider.java:128) ~[jersey-core-1.17.jar:1.17] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at com.sun.jersey.core.impl.provider.entity.StringProvider.writeTo(StringProvider.java:88) ~[jersey-core-1.17.jar:1.17] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at com.sun.jersey.core.impl.provider.entity.StringProvider.writeTo(StringProvider.java:58) ~[jersey-core-1.17.jar:1.17] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at com.sun.jersey.spi.container.ContainerResponse.write(ContainerResponse.java:306) ~[jersey-server-1.17.jar:1.17] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1479) ~[jersey-server-1.17.jar:1.17] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1391) ~[jersey-server-1.17.jar:1.17] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1381) ~[jersey-server-1.17.jar:1.17] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:416) ~[jersey-servlet-1.17.jar:1.17] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:538) ~[jersey-servlet-1.17.jar:1.17] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:716) ~[jersey-servlet-1.17.jar:1.17] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at javax.servlet.http.HttpServlet.service(HttpServlet.java:728) ~[servlet-api.jar:na] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:305) [catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210) [catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.access.intercept.FilterSecurityInterceptor.invoke(FilterSecurityInterceptor.java:108) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.access.intercept.FilterSecurityInterceptor.doFilter(FilterSecurityInterceptor.java:84) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:243) [catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210) [catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:330) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.access.intercept.FilterSecurityInterceptor.invoke(FilterSecurityInterceptor.java:118) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.access.intercept.FilterSecurityInterceptor.doFilter(FilterSecurityInterceptor.java:84) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.access.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:113) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.authentication.AnonymousAuthenticationFilter.doFilter(AnonymousAuthenticationFilter.java:113) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.servletapi.SecurityContextHolderAwareRequestFilter.doFilter(SecurityContextHolderAwareRequestFilter.java:154) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.context.request.async.WebAsyncManagerIntegrationFilter.doFilterInternal(WebAsyncManagerIntegrationFilter.java:50) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107) [spring-web-4.0.8.RELEASE.jar:4.0.8.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.authentication.www.BasicAuthenticationFilter.doFilter(BasicAuthenticationFilter.java:201) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.fineract.infrastructure.security.filter.TenantAwareBasicAuthenticationFilter.doFilter(TenantAwareBasicAuthenticationFilter.java:150) ~[TenantAwareBasicAuthenticationFilter.class:na] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.context.SecurityContextPersistenceFilter.doFilter(SecurityContextPersistenceFilter.java:87) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.access.channel.ChannelProcessingFilter.doFilter(ChannelProcessingFilter.java:144) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.FilterChainProxy.doFilterInternal(FilterChainProxy.java:192) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.security.web.FilterChainProxy.doFilter(FilterChainProxy.java:160) ~[spring-security-web-3.2.5.RELEASE.jar:3.2.5.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:243) [catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210) [catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.boot.context.web.ErrorPageFilter.doFilter(ErrorPageFilter.java:108) [spring-boot-1.1.6.RELEASE.jar:1.1.6.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.boot.context.web.ErrorPageFilter.access$000(ErrorPageFilter.java:58) [spring-boot-1.1.6.RELEASE.jar:1.1.6.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.boot.context.web.ErrorPageFilter$1.doFilterInternal(ErrorPageFilter.java:87) [spring-boot-1.1.6.RELEASE.jar:1.1.6.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107) [spring-web-4.0.8.RELEASE.jar:4.0.8.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.springframework.boot.context.web.ErrorPageFilter.doFilter(ErrorPageFilter.java:100) [spring-boot-1.1.6.RELEASE.jar:1.1.6.RELEASE] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:243) [catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210) [catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:222) [catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:123) [catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:502) [catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:171) [catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:99) [catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:953) [catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:118) [catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:408) [catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1023) [tomcat-coyote.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:589) [tomcat-coyote.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:312) [tomcat-coyote.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at java.lang.Thread.run(Thread.java:745) [na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: Caused by: java.net.SocketException: Broken pipe Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at java.net.SocketOutputStream.socketWrite0(Native Method) ~[na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:109) ~[na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at java.net.SocketOutputStream.write(SocketOutputStream.java:153) ~[na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at sun.security.ssl.OutputRecord.writeBuffer(OutputRecord.java:431) ~[na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at sun.security.ssl.OutputRecord.write(OutputRecord.java:417) ~[na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at sun.security.ssl.SSLSocketImpl.writeRecordInternal(SSLSocketImpl.java:864) ~[na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at sun.security.ssl.SSLSocketImpl.writeRecord(SSLSocketImpl.java:835) ~[na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at sun.security.ssl.AppOutputStream.write(AppOutputStream.java:123) ~[na:1.8.0_45-internal] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.coyote.http11.InternalOutputBuffer.realWriteBytes(InternalOutputBuffer.java:215) ~[tomcat-coyote.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.tomcat.util.buf.ByteChunk.flushBuffer(ByteChunk.java:480) ~[tomcat-coyote.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.tomcat.util.buf.ByteChunk.append(ByteChunk.java:366) ~[tomcat-coyote.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.coyote.http11.InternalOutputBuffer$OutputStreamOutputBuffer.doWrite(InternalOutputBuffer.java:240) ~[tomcat-coyote.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.coyote.http11.filters.ChunkedOutputFilter.doWrite(ChunkedOutputFilter.java:119) ~[tomcat-coyote.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.coyote.http11.AbstractOutputBuffer.doWrite(AbstractOutputBuffer.java:192) ~[tomcat-coyote.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.coyote.Response.doWrite(Response.java:517) ~[tomcat-coyote.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011at org.apache.catalina.connector.OutputBuffer.realWriteBytes(OutputBuffer.java:401) ~[catalina.jar:7.0.42] Aug 1 03:38:31 ip-172-31-7-48 35f2f10ad389[1056]: #011... 76 common frames omitted
