Also, this is how the Solr node's log looks when the above Curl request
fails:
2249660 [qtp883065319-18] INFO org.apache.solr.servlet.SolrDispatchFilter –
[admin] webapp=null path=/admin/collections
params={indent=on&action=CLUSTERSTATUS&wt=json} status=500 QTime=180028
2249657 [zkCallback-2-thread-12] INFO
org.apache.solr.cloud.DistributedQueue – LatchChildWatcher fired on path:
/overseer/collection-queue-work state: SyncConnected type
NodeChildrenChanged2249662 [qtp883065319-18] ERROR
org.apache.solr.servlet.SolrDispatchFilter –
null:org.apache.solr.common.SolrException: CLUSTERSTATUS the collection
time out:180s at
org.apache.solr.handler.admin.CollectionsHandler.handleResponse(CollectionsHandler.java:368)
at
org.apache.solr.handler.admin.CollectionsHandler.handleResponse(CollectionsHandler.java:320)
at
org.apache.solr.handler.admin.CollectionsHandler.handleClusterStatus(CollectionsHandler.java:640)
at
org.apache.solr.handler.admin.CollectionsHandler.handleRequestBody(CollectionsHandler.java:220)
at
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)
at
org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:729)
at
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:267)
at
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:207)
at
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
at
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
at
org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:557)
at
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:231)
at
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1075)
at
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
at
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
at
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
at
org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:255)
at
org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:154)
at
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
at org.eclipse.jetty.server.Server.handle(Server.java:368) at
org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
at
org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
at
org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640) at
org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235) at
org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82)
at org.eclipse.jetty.io.nio.SslConnection.handle(SslConnection.java:196) at
org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:628)
at
org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)2249666 [qtp883065319-18] ERROR
org.apache.solr.servlet.SolrDispatchFilter –
null:org.eclipse.jetty.io.EofException at
org.eclipse.jetty.http.HttpGenerator.flushBuffer(HttpGenerator.java:914) at
org.eclipse.jetty.http.AbstractGenerator.flush(AbstractGenerator.java:443)
at org.eclipse.jetty.server.HttpOutput.flush(HttpOutput.java:100) at
org.eclipse.jetty.server.AbstractHttpConnection$Output.flush(AbstractHttpConnection.java:1094)
at sun.nio.cs.StreamEncoder.implFlush(StreamEncoder.java:297) at
sun.nio.cs.StreamEncoder.flush(StreamEncoder.java:141) at
java.io.OutputStreamWriter.flush(OutputStreamWriter.java:229) at
org.apache.solr.util.FastWriter.flush(FastWriter.java:137) at
org.apache.solr.servlet.SolrDispatchFilter.writeResponse(SolrDispatchFilter.java:766)
at
org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:736)
at
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:267)
at
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:207)
at
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
at
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
at
org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:557)
at
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:231)
at
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1075)
at
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
at
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
at
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
at
org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:255)
at
org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:154)
at
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
at org.eclipse.jetty.server.Server.handle(Server.java:368) at
org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
at
org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
at
org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640) at
org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235) at
org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82)
at org.eclipse.jetty.io.nio.SslConnection.handle(SslConnection.java:196) at
org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:628)
at
org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745) Caused by: java.io.IOException:
Broken pipe at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at
sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:47) at
sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at
sun.nio.ch.IOUtil.write(IOUtil.java:65) at
sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:487) at
org.eclipse.jetty.io.nio.ChannelEndPoint.flush(ChannelEndPoint.java:293) at
org.eclipse.jetty.io.nio.SelectChannelEndPoint.flush(SelectChannelEndPoint.java:362)
at org.eclipse.jetty.io.nio.SslConnection.process(SslConnection.java:337)
at org.eclipse.jetty.io.nio.SslConnection.access$900(SslConnection.java:48)
at
org.eclipse.jetty.io.nio.SslConnection$SslEndPoint.flush(SslConnection.java:690)
at
org.eclipse.jetty.io.nio.SslConnection$SslEndPoint.flush(SslConnection.java:697)
at org.eclipse.jetty.http.HttpGenerator.flushBuffer(HttpGenerator.java:838)
... 37 more 2249675 [qtp883065319-18] WARN
org.eclipse.jetty.server.Response – Committed before 500 {msg=Broken
pipe,trace=org.eclipse.jetty.io.EofException at
org.eclipse.jetty.http.HttpGenerator.flushBuffer(HttpGenerator.java:914) at
org.eclipse.jetty.http.AbstractGenerator.flush(AbstractGenerator.java:443)
at org.eclipse.jetty.server.HttpOutput.flush(HttpOutput.java:100) at
org.eclipse.jetty.server.AbstractHttpConnection$Output.flush(AbstractHttpConnection.java:1094)
at sun.nio.cs.StreamEncoder.implFlush(StreamEncoder.java:297) at
sun.nio.cs.StreamEncoder.flush(StreamEncoder.java:141) at
java.io.OutputStreamWriter.flush(OutputStreamWriter.java:229) at
org.apache.solr.util.FastWriter.flush(FastWriter.java:137) at
org.apache.solr.servlet.SolrDispatchFilter.writeResponse(SolrDispatchFilter.java:766)
at
org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:736)
at
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:267)
at
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:207)
at
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
at
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
at
org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:557)
at
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:231)
at
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1075)
at
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
at
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
at
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
at
org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:255)
at
org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:154)
at
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
at org.eclipse.jetty.server.Server.handle(Server.java:368) at
org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
at
org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
at
org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640) at
org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235) at
org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82)
at org.eclipse.jetty.io.nio.SslConnection.handle(SslConnection.java:196) at
org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:628)
at
org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745) Caused by: java.io.IOException:
Broken pipe at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at
sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:47) at
sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at
sun.nio.ch.IOUtil.write(IOUtil.java:65) at
sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:487) at
org.eclipse.jetty.io.nio.ChannelEndPoint.flush(ChannelEndPoint.java:293) at
org.eclipse.jetty.io.nio.SelectChannelEndPoint.flush(SelectChannelEndPoint.java:362)
at org.eclipse.jetty.io.nio.SslConnection.process(SslConnection.java:337)
at org.eclipse.jetty.io.nio.SslConnection.access$900(SslConnection.java:48)
at
org.eclipse.jetty.io.nio.SslConnection$SslEndPoint.flush(SslConnection.java:690)
at
org.eclipse.jetty.io.nio.SslConnection$SslEndPoint.flush(SslConnection.java:697)
at org.eclipse.jetty.http.HttpGenerator.flushBuffer(HttpGenerator.java:838)
... 37 more,code=500}
On Mon Feb 09 2015 at 7:49:24 PM Avanish Raju <yar...@gmail.com> wrote:
> Hi all,
>
> I'm using self-signed certificates between my SolrCloud (4.10.3
> <http://mirror.sdunix.com/apache/lucene/solr/ref-guide/apache-solr-ref-guide-4.10.pdf>)
> instance and Curl/Solr http client with needClientAuth=true on jetty.xml.
> I'm able to load the Solr home page, and also make select/query calls and
> they work fine.
>
> However, when I make any Collections API calls, Basically any of the calls
> that are based on "https://localhost:8984/solr/admin/collections?action=";
> even using Curl locally, and a .pem file, some of the requests work, and
> some of them(about 1 in 3 or more) hang indefinitely.
>
> Has anyone run into this? How can I fix this?
>
> Thanks,
> Avanish
>
> PS: If I switch to the SocketConnector and use HTTP, I never see the
> hangs. Also, in both modes, my CPU uitilization is well below 15%. As for
> memory, I have 3.7 GB total of which more than 500 MB is free even when I'm
> making any requests to Solr
>
> This is how my CURL output looks for a request that's hanging until I hit
> Ctrl+C:
> curl -v -E solr-ssl.pem:secret --cacert solr-ssl.pem "
> https://localhost:8984/solr/admin/collections?action=CLUSTERSTATUS&wt=json&indent=on
> "
> * Hostname was NOT found in DNS cache* Trying 127.0.0.1...* Connected to
> localhost (127.0.0.1) port 8984 (#0)* successfully set certificate verify
> locations:* CAfile: solr-ssl.pem CApath: /etc/ssl/certs* SSLv3, TLS
> handshake, Client hello (1):* SSLv3, TLS handshake, Server hello (2):*
> SSLv3, TLS handshake, CERT (11):* SSLv3, TLS handshake, Server key
> exchange (12):* SSLv3, TLS handshake, Request CERT (13):* SSLv3, TLS
> handshake, Server finished (14):* SSLv3, TLS handshake, CERT (11):*
> SSLv3, TLS handshake, Client key exchange (16):* SSLv3, TLS handshake,
> CERT verify (15):* SSLv3, TLS change cipher, Client hello (1):* SSLv3,
> TLS handshake, Finished (20):* SSLv3, TLS change cipher, Client hello (1):*
> SSLv3, TLS handshake, Finished (20):* SSL connection using
> ECDHE-RSA-AES256-SHA384* Server certificate:* subject: C=Country;
> ST=State; L=Location; O=Organization; OU=Organizational Unit; CN=
> 36.38.154.104.bc.googleusercontent.com* start date: 2015-02-08 12:29:46
> GMT* expire date: 2042-06-25 12:29:46 GMT* subjectAltName: localhost
> matched* issuer: C=Country; ST=State; L=Location; O=Organization;
> OU=Organizational Unit; CN=36.38.154.104.bc.googleusercontent.com* SSL
> certificate verify ok.> GET
> /solr/admin/collections?action=CLUSTERSTATUS&wt=json&indent=on HTTP/1.1>
> User-Agent: curl/7.35.0> Host: localhost:8984> Accept: */*>
>
>
