So I'm running CouchDB 1.1.0 on RHEL 5.x. It took me a little while to
figure out exactly what was going on, but I seem to be experiencing
https://issues.apache.org/jira/browse/COUCHDB-994.
I have a database with just shy of 20 million documents and a few large-ish
views (~40GB uncompacted). Compacting the views is intermittently causing
crashes, sometimes it works, other times not. After the crash, accessing
the view seems to hang–no results, but the indexing doesn't restart
immediately. After a while, the index restarts, which obviously takes some
time.
Any help is appreciated!
The error thrown is:
[Mon, 01 Aug 2011 19:43:38 GMT] [info] [<0.293.0>] View index compaction
complete for jobs _design/by_tgusername_resource_date
[Mon, 01 Aug 2011 19:43:43 GMT] [error] [<0.293.0>] ** Generic server
<0.293.0> terminating
** Last message in was {'$gen_cast', {compact_done,
{group,
<<150,66,46,165,88,94,16,216,55,239,143,167,186,120,
139,10>>,
{db,<0.212.0>,<0.213.0>,nil,<<"1312208176813683">>,
<0.210.0>,<0.214.0>,
{db_header,5,19140068,0,
{15111154697,{19140048,18}},
{15111156806,19140066},
{15111159919,[]},
0,nil,15089743285,1000},
19140068,
{btree,<0.210.0>,
{15111154697,{19140048,18}},
#Fun<couch_db_updater.10.19222179>,
#Fun<couch_db_updater.11.21515767>,
#Fun<couch_btree.5.124754102>,
#Fun<couch_db_updater.12.93888648>},
{btree,<0.210.0>,
{15111156806,19140066},
#Fun<couch_db_updater.13.40165027>,
#Fun<couch_db_updater.14.82810239>,
#Fun<couch_btree.5.124754102>,
#Fun<couch_db_updater.15.104121193>},
{btree,<0.210.0>,
{15111159919,[]},
#Fun<couch_btree.0.83553141>,
#Fun<couch_btree.1.30790806>,
#Fun<couch_btree.2.124754102>,nil},
19140068,<<"jobs">>,
"/usr/local/var/lib/couchdb/jobs.couch",[],
[{<<"admins">>,
{[{<<"names">>,[<<"mrhanlon">>]},
{<<"roles">>,[]}]}},
{<<"readers">>,
{[{<<"names">>,[<<"portal">>]},
{<<"roles">>,[]}]}}],
15089743285,
{user_ctx,null,[],undefined},
nil,1000,
[before_header,after_header,on_file_open],
false},
<0.32596.3>,
<<"_design/by_tgusername_resource_date">>,
<<"erlang">>,[],
[{view,0,0,0,[], <<"fun ({Doc}) ->\n K =
proplists:get_value(<<\"tgup_username\">>, Doc, null),\n
M = proplists:get_value(<<\"resource_name\">>, Doc, null),\n N =
proplists:get_value(<<\"submit_time\">>, Do
c, null),\n Emit([K,M,list_to_binary(string:substr(binary_to_list(N),
1, 7))], null)\n end.">>,
{btree,<0.32596.3>,
{4360884819,{19140041,[19140041]}},
#Fun<couch_btree.3.83553141>,
#Fun<couch_btree.4.30790806>,
#Fun<couch_view.less_json_ids.2>,
#Fun<couch_view_group.10.110034783>},
[{<<"jobs">>,<<"_count">>}],
[]},
{view,1,0,0,[],
<<"fun ({Doc}) ->\n K =
proplists:get_value(<<\"tgup_username\">>, Doc, null),\n M =
proplists:get_value(<<\"resource_name\">>, Doc, null),\n N =
proplists:get_value(<<\"submit_time\">>, Doc, null),\n S =
proplists:get_value(<<\"sus\">>, Doc, null),\n
Emit([K,M,list_to_binary(string:substr(binary_to_list(N), 1, 7))], S)\n
end.">>,
{btree,<0.32596.3>,
{6582607835,{19140041,[3655450218.1625037]}},
#Fun<couch_btree.3.83553141>,
#Fun<couch_btree.4.30790806>,
#Fun<couch_view.less_json_ids.2>,
#Fun<couch_view_group.10.110034783>},
[{<<"sus">>,<<"_sum">>}],
[]}],
{[]},
{btree,<0.32596.3>,
{2721584482,[]},
#Fun<couch_btree.0.83553141>,
#Fun<couch_btree.1.30790806>,
#Fun<couch_btree.2.124754102>,nil},
19140068,0,nil,nil}}}
** When Server state == {group_state,undefined,<<"jobs">>,
{"/usr/local/var/lib/couchdb",<<"jobs">>,
{group,
<<150,66,46,165,88,94,16,216,55,239,143,167,186,120,
139,10>>,
nil,nil,<<"_design/by_tgusername_resource_date">>,
<<"erlang">>,[],
[{view,0,0,0,[],
<<"fun ({Doc}) ->\n K =
proplists:get_value(<<\"tgup_username\">>, Doc, null),\n M =
proplists:get_value(<<\"resource_name\">>, Doc, null),\n N =
proplists:get_value(<<\"submit_time\">>, Doc, null),\n
Emit([K,M,list_to_binary(string:substr(binary_to_list(N), 1, 7))], null)\n
end.">>,
nil,
[{<<"jobs">>,<<"_count">>}],
[]},
{view,1,0,0,[],
<<"fun ({Doc}) ->\n K =
proplists:get_value(<<\"tgup_username\">>, Doc, null),\n M =
proplists:get_value(<<\"resource_name\">>, Doc, null),\n N =
proplists:get_value(<<\"submit_time\">>, Doc, null),\n S =
proplists:get_value(<<\"sus\">>, Doc, null),\n
Emit([K,M,list_to_binary(string:substr(binary_to_list(N), 1, 7))], S)\n
end.">>,
nil,
[{<<"sus">>,<<"_sum">>}],
[]}],
{[]},
nil,0,0,nil,nil}},
{group,
<<150,66,46,165,88,94,16,216,55,239,143,167,186,120,
139,10>>,
nil,<0.294.0>,
<<"_design/by_tgusername_resource_date">>,
<<"erlang">>,[],
[{view,0,19140068,0,[],
<<"fun ({Doc}) ->\n K =
proplists:get_value(<<\"tgup_username\">>, Doc, null),\n M =
proplists:get_value(<<\"resource_name\">>, Doc, null),\n N =
proplists:get_value(<<\"submit_time\">>, Doc, null),\n
Emit([K,M,list_to_binary(string:substr(binary_to_list(N), 1, 7))], null)\n
end.">>,
{btree,<0.294.0>,
{38019649283,{19140041,[19140041]}},
#Fun<couch_btree.3.83553141>,
#Fun<couch_btree.4.30790806>,
#Fun<couch_view.less_json_ids.2>,
#Fun<couch_view_group.10.110034783>},
[{<<"jobs">>,<<"_count">>}],
[]},
{view,1,19140068,0,[],
<<"fun ({Doc}) ->\n K =
proplists:get_value(<<\"tgup_username\">>, Doc, null),\n M =
proplists:get_value(<<\"resource_name\">>, Doc, null),\n N =
proplists:get_value(<<\"submit_time\">>, Doc, null),\n S =
proplists:get_value(<<\"sus\">>, Doc, null),\n
Emit([K,M,list_to_binary(string:substr(binary_to_list(N), 1, 7))], S)\n
end.">>,
{btree,<0.294.0>,
{38019866647,{19140041,[3655450218.1625037]}},
#Fun<couch_btree.3.83553141>,
#Fun<couch_btree.4.30790806>,
#Fun<couch_view.less_json_ids.2>,
#Fun<couch_view_group.10.110034783>},
[{<<"sus">>,<<"_sum">>}],
[]}],
{[]},
{btree,<0.294.0>,
{38019458154,[]},
#Fun<couch_btree.0.83553141>,
#Fun<couch_btree.1.30790806>,
#Fun<couch_btree.2.124754102>,nil},
19140068,0,nil,nil},
nil,<0.32598.3>,false,[],<0.6422.0>}
** Reason for termination ==
** {timeout,{gen_server,call,[<0.6422.0>,{drop,<0.293.0>}]}}
[Mon, 01 Aug 2011 19:43:44 GMT] [error] [<0.293.0>] {error_report,<0.29.0>,
{<0.293.0>,crash_report,
[[{initial_call,{couch_view_group,init,['Argument__1']}},
{pid,<0.293.0>},
{registered_name,[]},
{error_info,
{exit,
{timeout,
{gen_server,call,[<0.6422.0>,{drop,<0.293.0>}]}},
[{gen_server,terminate,6},
{proc_lib,init_p_do_apply,3}]}},
{ancestors,[<0.292.0>]},
{messages,[]},
{links,[<0.110.0>,<0.32596.3>]},
{dictionary,[]},
{trap_exit,true},
{status,running},
{heap_size,6765},
{stack_size,24},
{reductions,61453380}],
[]]}}
[Mon, 01 Aug 2011 19:43:49 GMT] [error] [<0.32596.3>] ** Generic server
<0.32596.3> terminating
** Last message in was {'EXIT',<0.293.0>,
{timeout,
{gen_server,call,
[<0.6422.0>,{drop,<0.293.0>}]}}}
** When Server state ==
{file,{file_descriptor,prim_file,{#Port<0.3562>,29}},
0,6582608978}
** Reason for termination ==
** {timeout,{gen_server,call,[<0.6422.0>,{drop,<0.293.0>}]}}
[Mon, 01 Aug 2011 19:43:49 GMT] [error] [<0.32596.3>]
{error_report,<0.29.0>,
{<0.32596.3>,crash_report,
[[{initial_call,{couch_file,init,['Argument__1']}},
{pid,<0.32596.3>},
{registered_name,[]},
{error_info,
{exit,
{timeout,
{gen_server,call,[<0.6422.0>,{drop,<0.293.0>}]}},
[{gen_server,terminate,6},
{proc_lib,init_p_do_apply,3}]}},
{ancestors,[<0.293.0>,<0.292.0>]},
{messages,[]},
{links,[]},
{dictionary,[]},
{trap_exit,true},
{status,running},
{heap_size,987},
{stack_size,24},
{reductions,641719449}],
[]]}}
Regards,
-Matt.
--
Matthew Rollins Hanlon
http://squareoftwo.com
---------------------
http://wikipedia.org/wiki/Hanlon's_razor
