Hi Luke, Here is some error log from solr.log.
2017-02-04 13:15:15,420 [ERROR] <qtp775533998-16>@SolrException.java:109 org.apache.solr.common.SolrException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://nosql-2.dsdb:8093/internal_solr/production_scheduling at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:308) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135) at org.apache.solr.core.SolrCore.execute(SolrCore.java:1916) at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:780) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:427) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:217) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:557) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:231) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1075) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:255) at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:154) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116) at org.eclipse.jetty.server.Server.handle(Server.java:368) at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489) at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53) at org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:953) at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1014) at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:861) at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240) at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72) at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://nosql-2.dsdb:8093/internal_solr/production_scheduling at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:507) at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:199) at org.apache.solr.handler.component.HttpShardHandler$1.call(HttpShardHandler.java:156) at org.apache.solr.handler.component.HttpShardHandler$1.call(HttpShardHandler.java:118) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) ... 1 more Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143) at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57) at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260) at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283) at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251) at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197) at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682) at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486) at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57) at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:395) ... 9 more 2017-02-04 13:15:15,431 [ERROR] <qtp775533998-16>@SolrException.java:120 null:org.apache.solr.common.SolrException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://nosql-2.dsdb:8093/internal_solr/production_scheduling at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:308) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135) at org.apache.solr.core.SolrCore.execute(SolrCore.java:1916) at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:780) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:427) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:217) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:557) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:231) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1075) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:255) at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:154) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116) at org.eclipse.jetty.server.Server.handle(Server.java:368) at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489) at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53) at org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:953) at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1014) at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:861) at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240) at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72) at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://nosql-2.dsdb:8093/internal_solr/production_scheduling at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:507) at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:199) at org.apache.solr.handler.component.HttpShardHandler$1.call(HttpShardHandler.java:156) at org.apache.solr.handler.component.HttpShardHandler$1.call(HttpShardHandler.java:118) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) ... 1 more Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143) at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57) at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260) at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283) at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251) at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197) at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682) at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486) at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57) at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:395) ... 9 more Br, Alex 2017-02-04 23:30 GMT+08:00 Luke Bakken <[email protected]>: > Hi Alex - > > What is in solr.log ? > -- > Luke Bakken > Engineer > [email protected] > > > On Sat, Feb 4, 2017 at 3:37 AM, Alex Feng <[email protected]> wrote: > > Hello Riak users, > > > > I recently found our solr system got down after running some time, it got > > recovered after a restart. But then it happened again. > > > > Below is the error output, do you guys have any clue about this ? > > > > > > 2017-02-04 18:12:22.329 [error] <0.830.0>@yz_kv:index_internal:237 > failed to > > index object > > {{<<"process_test_result">>,<<83,104,105,76,111,110,103,90,105,9 > > 5,77,66,80,65,67,75,45,231,148,159,228,186,167,232,189, > 166,233,151,180,50,48,49,55,50,52>>},<<"9H5AlII0OZm8xwnGbI">>} > > with error {"Failed to index docs",o > > ther,{error,{conn_failed,{error,econnrefused}}}} because > > [{yz_solr,index,3,[{file,"src/yz_solr.erl"},{line,205}]},{ > yz_kv,index,7,[{file,"src/yz_kv.erl"},{ > > line,293}]},{yz_kv,index_internal,5,[{file,"src/yz_kv. > erl"},{line,224}]},{riak_kv_vnode,actual_put,6,[{file," > src/riak_kv_vnode.erl"},{line,1619}]},{riak_k > > v_vnode,perform_put,3,[{file,"src/riak_kv_vnode.erl"},{line, > 1607}]},{riak_kv_vnode,do_put,7,[{file,"src/riak_kv_vnode. > erl"},{line,1398}]},{riak_kv_vnode,h > > andle_command,3,[{file,"src/riak_kv_vnode.erl"},{line,558} > ]},{riak_core_vnode,vnode_command,3,[{file,"src/riak_ > core_vnode.erl"},{line,346}]}] > > > > > > > > restart node, got this> > > > > > > 2017-02-04 18:13:47.026 [error] <0.4340.0>@yz_pb_search:maybe_ > process:111 > > {solr_error,{500,"http://localhost:8093/internal_solr/ > process_test_result/select > > ",<<"{\"error\":{\"msg\":\"org.apache.solr.client.solrj. > SolrServerException: > > Server refused connection at: > > http://nosql-3.dsdb:8093/internal_solr/process_ > > test_result\",\"trace\":\"org.apache.solr.common.SolrException: > > org.apache.solr.client.solrj.SolrServerException: Server refused > connection > > at: http://nos > > ql-3.dsdb:8093/internal_solr/process_test_result\\n\\tat > > org.apache.solr.handler.component.SearchHandler.handleRequestBody( > SearchHandler.java:308)\\n\\tat > > > > org.apache.solr.handler.RequestHandlerBase.handleRequest( > RequestHandlerBase.java:135)\\n\\tat > > org.apache.solr.core.SolrCore.execute(SolrCore.java:1916)\\ > > n\\tat > > org.apache.solr.servlet.SolrDispatchFilter.execute( > SolrDispatchFilter.java:780)\\n\\tat > > org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDis > > patchFilter.java:427)\\n\\tat > > org.apache.solr.servlet.SolrDispatchFilter.doFilter( > SolrDispatchFilter.java:217)\\n\\tat > > org.eclipse.jetty.servlet.ServletHa > > ndler$CachedChain.doFilter(ServletHandler.java:1419)\\n\\tat > > org.eclipse.jetty.servlet.ServletHandler.doHandle( > ServletHandler.java:455)\\n\\tat > > org.eclipse.jetty.server.handler.ScopedHandler.handle( > ScopedHandler.java:137)\\n\\tat > > org.eclipse.jetty.security.SecurityHandler.handle( > SecurityHandler.java:557)\\n\\tat > > org.eclipse.jetty.server.session.SessionHandler. > doHandle(SessionHandler.java:231)\\n\\tat > > org.eclipse.jetty.server.handler.ContextHandler. > doHandle(ContextHandler.java:1075)\\n\\tat > > org.eclipse.jetty.servlet.ServletHandler.doScope( > ServletHandler.java:384)\\n\\tat > > org.eclipse.jetty.server.session.SessionHandler. > doScope(SessionHandler.java:193)\\n\\tat > > org.eclipse.jetty.server.handler.ContextHandler. > doScope(ContextHandler.java:1009)\\n\\tat > > org.eclipse.jetty.server.handler.ScopedHandler.handle( > ScopedHandler.java:135)\\n\\tat > > org.eclipse.jetty.server.handler.ContextHandlerCollection.handle( > ContextHandlerCollection.java:255)\\n\\tat > > org.eclipse.jetty.server.handler.HandlerCollection. > handle(HandlerCollection.java:154)\\n\\tat > > org.eclipse.jetty.server.handler.HandlerWrap...">>}} > > [{yz_solr,search,3,[{file,"src/yz_solr.erl"},{line,292}]} > ,{yz_pb_search,maybe_process,3,[{file,"src/yz_pb_search. > erl"},{line,76}]},{riak_api_pb_server,process_message,4,[{ > file,"src/riak_api_pb_server.erl"},{line,388}]},{riak_api_ > pb_server,connected,2,[{file,"src/riak_api_pb_server.erl"},{ > line,226}]},{riak_api_pb_server,decode_buffer,2,[{file, > "src/riak_api_pb_server.erl"},{line,364}]},{gen_fsm,handle_ > msg,7,[{file,"gen_fsm.erl"},{line,505}]},{proc_lib,init_p_ > do_apply,3,[{file,"proc_lib.erl"},{line,239}]}] > > 2017-02-04 18:17:02.756 [error] > > <0.6871.0>@riak_core_handoff_receiver:handle_info:97 Handoff receiver > for > > partition 0 exited abnormally after processing 0 objects from > > {"192.168.9.247",39483}: > > {error,{vnode_timeout,60000,844,<<131,104,2,100,0,10,101, > 110,99,111,100,101,95,114,97,119,104,3,104,2,109,0,0,0,19, > 112,114,111,99,101,115,115,95,116,101,115,116,95,114,101, > 115,117,108,116,109,0,0,0,35,83,104,105,76,111,110,103,90, > 105,95,77,66,80,65,67,75,45,231,148,159,228,186,167,232, > 189,166,233,151,180,50,48,49,55,50,52,109,0,0,0,18,57,72, > 53,65,109,84,122,54,79,109,99,108,87,107,87,68,80,85,109,0, > 0,2,220,53,1,0,0,0,38,131,108,0,0,0,1,104,2,109,0>>}} > > > > > > Br, > > > > Alex > > > > > > _______________________________________________ > > riak-users mailing list > > [email protected] > > http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com > > >
_______________________________________________ riak-users mailing list [email protected] http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com
