I bet you're hitting a bug we just recently fixed on trunk. Basically, there was a possibility that errors in some of the JS functions would end up causing a couchjs process to be come unusable without removing it from the list. Eventually there wouldn't be any spots left and clients would get timeouts like you see.
Patch is at [1]. If it doesn't apply cleanly, you really only need the bits from couch_os_process.erl and couch_query_servers.erl. The rest is just test code. https://github.com/apache/couchdb/commit/95da6f6f4246d2e8e86a3cf92ddf6487e46c10e9 On Wed, Aug 24, 2011 at 3:42 PM, Chris Stockton <[email protected]> wrote: > We have about 3000 active databases on our server, about 200 of those > users (most active ones) keep randomly failing to load views. I am > having difficulty trying to decipher this stack trace to make any > sense of it, does anyone have any suggestions about what is happening > here? Are we just reaching some sort of capacity limit? The machines > are Intel(R) Xeon(R) CPU L5520 @ 2.27GHz x16 core with 50mb ram, we > are fine on disk space, and requests per second to be honest is around > 1 with plenty of idle time between little spurts. > > -Chris > > ---------------------------------------- > > [Wed, 24 Aug 2011 15:36:31 GMT] [debug] [<0.14283.217>] 'GET' > /db_946/_design/views/_view/getByType?skip=0&key=["model","M4e551a78a9b3d"]&limit=25 > {1, > > 1} from "WITHHELD" > Headers: [{'Accept',"*/*"}, > {'Authorization',"Basic WITHHELD"}, > {'Content-Type',"application/json; charset=utf-8"}, > {'Host',"WITHHELD:5984"}, > {'User-Agent',"PECL::HTTP/1.7.0 (PHP/5.3.4)"}] > [Wed, 24 Aug 2011 15:36:31 GMT] [debug] [<0.14283.217>] OAuth Params: > [{"skip","0"}, > {"key", > "[\"model\",\"M4e551a78a9b3d\"]"}, > {"limit","25"}] > [Wed, 24 Aug 2011 15:36:31 GMT] [debug] [<0.14283.217>] request_group > {Pid, Seq} {<0.25836.0>,1077} > [Wed, 24 Aug 2011 15:36:31 GMT] [debug] [<0.82.0>] New task status for > db_946 _design/views: Processed 0 of 25 changes (0%) > [Wed, 24 Aug 2011 15:36:31 GMT] [debug] [<0.82.0>] New task status for > db_946 _design/views: Finishing. > [Wed, 24 Aug 2011 15:36:35 GMT] [debug] [<0.31520.215>] 'GET' > /_all_dbs {1,1} from "WITHHELD" > Headers: [{'Accept',"*/*"}, > {'Authorization',"Basic WITHHELD"}, > {'Content-Type',"application/json; charset=utf-8"}, > {'Host',"WITHHELD:5984"}, > {'User-Agent',"PECL::HTTP/1.7.0 (PHP/5.3.4)"}] > [Wed, 24 Aug 2011 15:36:35 GMT] [debug] [<0.31520.215>] OAuth Params: [] > [Wed, 24 Aug 2011 15:36:36 GMT] [debug] [<0.25836.0>] Exit from linked > pid: {<0.27488.217>, > {timeout, > {gen_server,call, > [couch_query_servers, > {get_proc, > <<"javascript">>}]}}} > [Wed, 24 Aug 2011 15:36:36 GMT] [error] [<0.25836.0>] ** Generic > server <0.25836.0> terminating > ** Last message in was {'EXIT',<0.27488.217>, > {timeout, > {gen_server,call, > [couch_query_servers, > {get_proc,<<"javascript">>}]}}} > ** When Server state == {group_state,undefined,<<"db_946">>, > {"/WITHHELD/5984/views",<<"db_946">>, > {group, > <<88,143,236,54,199,129,241,187,65,118,56,84,121,73, > 170,203>>, > nil,nil,<<"_design/views">>,<<"javascript">>,[], > [{view,0,0,0, > [<<"getByType">>], > <<"function(doc) {\n \n var sdoc = > eval(uneval(doc));\n if((typeof doc.model) !== \"undefined\") {\n > sdoc.data.rev = doc._rev;\n emit([doc.type, doc.model], > sdoc.data);\n } else if((typeof doc.type) > !== \"undefined\" && doc.type === \"couchimport\") {\n > emit([doc.type, doc.time], null);\n }\n}">>, > nil,[],[]}], > {[]}, > nil,0,0,nil,nil}}, > {group, > <<88,143,236,54,199,129,241,187,65,118,56,84,121,73, > 170,203>>, > {db,<0.6703.0>,<0.6717.0>,nil, > <<"1313769124135037">>,<0.24135.71>,<0.24137.71>, > {db_header,5,1077,0, > {1011747,{410,120}}, > {1014120,530}, > {1022521,[]}, > 0,nil,nil,1000}, > 1077, > {btree,<0.24135.71>, > {1011747,{410,120}}, > #Fun<couch_db_updater.10.118480628>, > #Fun<couch_db_updater.11.70256084>, > #Fun<couch_btree.5.49826377>, > #Fun<couch_db_updater.12.32819211>}, > {btree,<0.24135.71>, > {1014120,530}, > #Fun<couch_db_updater.13.25773661>, > #Fun<couch_db_updater.14.39338119>, > #Fun<couch_btree.5.49826377>, > #Fun<couch_db_updater.15.41342280>}, > {btree,<0.24135.71>, > {1022521,[]}, > #Fun<couch_btree.0.67022066>, > #Fun<couch_btree.1.72205677>, > #Fun<couch_btree.2.23775741>,nil}, > 1077,<<"db_946">>, > "/WITHHELD/5984/dbs/db_946.couch",[],[], > nil, > {user_ctx,null,[],undefined}, > nil,1000, > [before_header,after_header,on_file_open], > false}, > <0.24246.71>,<<"_design/views">>,<<"javascript">>, > [], > [{view,0,0,0, > [<<"getByType">>], > <<"function(doc) {\n \n var sdoc = > eval(uneval(doc));\n if((typeof doc.model) !== \"undefined\") {\n > sdoc.data.rev = doc._rev;\n emit([doc.type, doc.model], > sdoc.data);\n } else if((typeof doc.type) > !== \"undefined\" && doc.type === \"couchimport\") {\n > emit([doc.type, doc.time], null);\n }\n}">>, > {btree,<0.24246.71>, > {142166,{354,[]}}, > #Fun<couch_btree.3.22452524>, > #Fun<couch_btree.4.103377371>, > #Fun<couch_view.less_json_ids.2>, > #Fun<couch_view_group.10.1717152>}, > [],[]}], > {[]}, > {btree,<0.24246.71>, > {30445,[]}, > #Fun<couch_btree.0.67022066>, > #Fun<couch_btree.1.72205677>, > #Fun<couch_btree.2.23775741>,nil}, > 1051,0,nil,nil}, > <0.27488.217>,nil,false, > [{{<0.14283.217>,#Ref<0.0.1341.210504>},1077}], > <0.24251.71>} > ** Reason for termination == > ** {timeout,{gen_server,call, > [couch_query_servers,{get_proc,<<"javascript">>}]}} > > [Wed, 24 Aug 2011 15:36:36 GMT] [error] [<0.25836.0>] {error_report,<0.30.0>, > {<0.25836.0>,crash_report, > [[{initial_call, > {couch_view_group,init,['Argument__1']}}, > {pid,<0.25836.0>}, > {registered_name,[]}, > {error_info, > {exit, > {timeout, > {gen_server,call, > [couch_query_servers, > {get_proc,<<"javascript">>}]}}, > [{gen_server,terminate,6}, > {proc_lib,init_p_do_apply,3}]}}, > {ancestors,[<0.25835.0>]}, > {messages,[]}, > {links,[<0.24246.71>,<0.111.0>]}, > {dictionary,[]}, > {trap_exit,true}, > {status,running}, > {heap_size,6765}, > {stack_size,24}, > {reductions,10266}], > []]}} > [Wed, 24 Aug 2011 15:36:36 GMT] [debug] [<0.14283.217>] request_group > Error {timeout, > {gen_server,call, > [couch_query_servers, > {get_proc,<<"javascript">>}]}} > [Wed, 24 Aug 2011 15:36:36 GMT] [error] [<0.24246.71>] ** Generic > server <0.24246.71> terminating > ** Last message in was {'EXIT',<0.25836.0>, > {timeout, > {gen_server,call, > [couch_query_servers, > {get_proc,<<"javascript">>}]}}} > ** When Server state == {file, > {file_descriptor,prim_file,{#Port<0.53260>,7389}}, > 0,143464} > ** Reason for termination == > ** {timeout,{gen_server,call, > [couch_query_servers,{get_proc,<<"javascript">>}]}} > > [Wed, 24 Aug 2011 15:36:36 GMT] [error] [<0.24246.71>] {error_report,<0.30.0>, > {<0.24246.71>,crash_report, > [[{initial_call,{couch_file,init,['Argument__1']}}, > {pid,<0.24246.71>}, > {registered_name,[]}, > {error_info, > {exit, > {timeout, > {gen_server,call, > [couch_query_servers, > {get_proc,<<"javascript">>}]}}, > [{gen_server,terminate,6}, > {proc_lib,init_p_do_apply,3}]}}, > {ancestors,[<0.25836.0>,<0.25835.0>]}, > {messages,[]}, > {links,[<0.24251.71>]}, > {dictionary,[]}, > {trap_exit,true}, > {status,running}, > {heap_size,2584}, > {stack_size,24}, > {reductions,17469}], > [{neighbour, > [{pid,<0.24251.71>}, > {registered_name,[]}, > {initial_call, > {couch_ref_counter,init,['Argument__1']}}, > {current_function,{gen_server,loop,6}}, > {ancestors,[<0.25836.0>,<0.25835.0>]}, > {messages,[]}, > {links,[<0.24246.71>]}, > {dictionary,[]}, > {trap_exit,false}, > {status,waiting}, > {heap_size,377}, > {stack_size,9}, > {reductions,141}]}]]}} > [Wed, 24 Aug 2011 15:36:36 GMT] [debug] [<0.14283.217>] Minor error in > HTTP request: {timeout, > {gen_server,call, > [couch_query_servers, > {get_proc, > <<"javascript">>}]}} > [Wed, 24 Aug 2011 15:36:36 GMT] [debug] [<0.14283.217>] Stacktrace: > [{io_lib_pretty,cind_tag_tuple,7}, > {io_lib_pretty,while_fail,3}, > {io_lib_pretty,print,6}, > {io_lib_format,build,3}, > {io_lib_format,build,3}, > {io_lib_format,build,3}, > {io_lib_format,build,3}, > {io_lib_format,build,3}] > [Wed, 24 Aug 2011 15:36:36 GMT] [info] [<0.14283.217>] WITHHELD - - > 'GET' > /db_946/_design/views/_view/getByType?skip=0&key=["model","M4e551a78a9b3d"]&limit=25 > 500 > [Wed, 24 Aug 2011 15:36:36 GMT] [debug] [<0.14283.217>] httpd 500 > error response: > {"error":"timeout","reason":"{gen_server,call,[couch_query_servers,{get_proc,<<\"javascript\">>}]}"} >
