Hi Toby, this is a recently identified regression in 1.4.7. Go into your bitcask data directories, and you'll likely see a number of 0 byte data files and 18 byte hintfiles with the same file ID number. Stop the node, move those aside and you should be back in business. find can be used to automate the process. Another 1.4 release will be forthcoming to remedy the issue.
On Thu, Feb 6, 2014 at 4:20 PM, Toby Corkindale <[email protected]> wrote: > Our test environment Riak cluster is looking pretty unhealthy at the > moment, with quite a lot of crashes being reported. Is anyone able to > advise what the cause is? > > From error.log: > > [error] <0.13413.31> CRASH REPORT Process <0.13413.31> with 0 > neighbours exited with reason: no match of right hand value > {error,{badmatch,{error,eexist}}} in bitcask:do_put/5 line 1232 in > gen_server:terminate/6 line 747 > > > From crash.log: > > =ERROR REPORT==== > ** Generic server <0.5600.32> terminating > ** Last message in was > {'EXIT',<0.5599.32>,{{badmatch,{error,{badmatch,{error,eexist}}}},[{bitcask,do_put,5,[{file,"src/bitcask.erl"},{line,1232}]},{bitcask,put,3,[{file,"src/bitcask.erl"},{line,244}]},{riak_kv_bitcask_backend,put,5,[{file,"src/riak_kv_bitcask_backend.erl"},{line,168}]},{riak_cs_kv_multi_backend,put,5,[{file,"src/riak_cs_kv_multi_backend.erl"},{line,255}]},{riak_kv_vnode,encode_and_put,6,[{file,"src/riak_kv_vnode.erl"},{line,1776}]},{riak_kv_vnode,perform_put,3,[{file,"src/riak_kv_vnode.erl"},{line,1162}]},{riak_kv_vnode,do_put,7,[{file,"src/riak_kv_vnode.erl"},{line,1009}]},{riak_kv_vnode,handle_command,3,[{file,"src/riak_kv_vnode.erl"},{line,419}]}]}} > ** When Server state == > {state,{<0.5600.32>,poolboy_sup},simple_one_for_one,[{child,undefined,riak_core_vnode_worker,{riak_core_vnode_worker,start_link,[[{worker_module,riak_core_vnode_worker},{worker_args,[91343852333181432387730302044767688728495783936,[],worker_props,<0.5598.32>]},{worker_callback_mod,riak_kv_worker},{size,10},{max_overflow,0}]]},temporary,5000,worker,[riak_core_vnode_worker]}],{set,10,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},{{[<0.5603.32>],[<0.5604.32>],[<0.5605.32>],[<0.5606.32>],[<0.5607.32>],[<0.5608.32>],[<0.5609.32>],[<0.5610.32>],[],[],[],[],[],[],[<0.5601.32>],[<0.5602.32>]}}},0,1,[],poolboy_sup,{riak_core_vnode_worker,[{worker_module,riak_core_vnode_worker},{worker_args,[91343852333181432387730302044767688728495783936,[],worker_props,<0.5598.32>]},{worker_callback_mod,riak_kv_worker},{size,10},{max_overflow,0}]}} > ** Reason for termination == > ** > {{badmatch,{error,{badmatch,{error,eexist}}}},[{bitcask,do_put,5,[{file,"src/bitcask.erl"},{line,1232}]},{bitcask,put,3,[{file,"src/bitcask.erl"},{line,244}]},{riak_kv_bitcask_backend,put,5,[{file,"src/riak_kv_bitcask_backend.erl"},{line,168}]},{riak_cs_kv_multi_backend,put,5,[{file,"src/riak_cs_kv_multi_backend.erl"},{line,255}]},{riak_kv_vnode,encode_and_put,6,[{file,"src/riak_kv_vnode.erl"},{line,1776}]},{riak_kv_vnode,perform_put,3,[{file,"src/riak_kv_vnode.erl"},{line,1162}]},{riak_kv_vnode,do_put,7,[{file,"src/riak_kv_vnode.erl"},{line,1009}]},{riak_kv_vnode,handle_command,3,[{file,"src/riak_kv_vnode.erl"},{line,419}]}]} > =CRASH REPORT==== > crasher: > initial call: supervisor:poolboy_sup/1 > pid: <0.5600.32> > registered_name: [] > exception exit: > {{{badmatch,{error,{badmatch,{error,eexist}}}},[{bitcask,do_put,5,[{file,"src/bitcask.erl"},{line,1232}]},{bitcask,put,3,[{file,"src/bitcask.erl"},{line,244}]},{riak_kv_bitcask_backend,put,5,[{file,"src/riak_kv_bitcask_backend.erl"},{line,168}]},{riak_cs_kv_multi_backend,put,5,[{file,"src/riak_cs_kv_multi_backend.erl"},{line,255}]},{riak_kv_vnode,encode_and_put,6,[{file,"src/riak_kv_vnode.erl"},{line,1776}]},{riak_kv_vnode,perform_put,3,[{file,"src/riak_kv_vnode.erl"},{line,1162}]},{riak_kv_vnode,do_put,7,[{file,"src/riak_kv_vnode.erl"},{line,1009}]},{riak_kv_vnode,handle_command,3,[{file,"src/riak_kv_vnode.erl"},{line,419}]}]},[{gen_server,terminate,6,[{file,"gen_server.erl"},{line,747}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,227}]}]} > ancestors: > [<0.5599.32>,<0.5598.32>,<0.5479.32>,riak_core_vnode_sup,riak_core_sup,<0.154.0>] > messages: [] > links: [] > dictionary: [] > trap_exit: true > status: running > heap_size: 6765 > stack_size: 24 > reductions: 1945 > neighbours: > =ERROR REPORT==== > ** Generic server <0.11652.32> terminating > ** Last message in was > {'DOWN',#Ref<0.0.36.156577>,process,<0.5479.32>,{{badmatch,{error,{badmatch,{error,eexist}}}},[{bitcask,do_put,5,[{file,"src/bitcask.erl"},{line,1232}]},{bitcask,put,3,[{file,"src/bitcask.erl"},{line,244}]},{riak_kv_bitcask_backend,put,5,[{file,"src/riak_kv_bitcask_backend.erl"},{line,168}]},{riak_cs_kv_multi_backend,put,5,[{file,"src/riak_cs_kv_multi_backend.erl"},{line,255}]},{riak_kv_vnode,encode_and_put,6,[{file,"src/riak_kv_vnode.erl"},{line,1776}]},{riak_kv_vnode,perform_put,3,[{file,"src/riak_kv_vnode.erl"},{line,1162}]},{riak_kv_vnode,do_put,7,[{file,"src/riak_kv_vnode.erl"},{line,1009}]},{riak_kv_vnode,handle_command,3,[{file,"src/riak_kv_vnode.erl"},{line,419}]}]}} > ** When Server state == {state,undefined,undefined} > ** Reason for termination == > ** > {{badmatch,{error,badarg}},[{bitcask_file,handle_info,2,[{file,"src/bitcask_file.erl"},{line,170}]},{gen_server,handle_msg,5,[{file,"gen_server.erl"},{line,607}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,227}]}]} > 2014-02-07 11:12:41 =CRASH REPORT==== > crasher: > initial call: bitcask_file:init/1 > pid: <0.11652.32> > registered_name: [] > exception exit: > {{{badmatch,{error,badarg}},[{bitcask_file,handle_info,2,[{file,"src/bitcask_file.erl"},{line,170}]},{gen_server,handle_msg,5,[{file,"gen_server.erl"},{line,607}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,227}]}]},[{gen_server,terminate,6,[{file,"gen_server.erl"},{line,747}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,227}]}]} > ancestors: [<0.5479.32>,riak_core_vnode_sup,riak_core_sup,<0.154.0>] > messages: [] > links: [] > dictionary: [] > trap_exit: false > status: running > heap_size: 1597 > stack_size: 24 > reductions: 472 > neighbours: > =ERROR REPORT==== > ** State machine <0.11640.32> terminating > ** Last event in was timeout > ** When State == key_exchange > ** Data == > {state,{91343852333181432387730302044767688728495783936,'[email protected]'},{114179815416476790484662877555959610910619729920,'[email protected]'},{91343852333181432387730302044767688728495783936,3},<0.5596.32>,<9663.606.0>,1,300000} > ** Reason for termination = > ** > {normal,{gen_server,call,[<0.5596.32>,{compare,{91343852333181432387730302044767688728495783936,3},#Fun<riak_kv_exchange_fsm.7.96623222>,#Fun<riak_kv_exchange_fsm.8.96623222>},infinity]}} > 2014-02-07 11:12:41 =CRASH REPORT==== > crasher: > initial call: riak_kv_exchange_fsm:init/1 > pid: <0.11640.32> > registered_name: [] > exception exit: > {{normal,{gen_server,call,[<0.5596.32>,{compare,{91343852333181432387730302044767688728495783936,3},#Fun<riak_kv_exchange_fsm.7.96623222>,#Fun<riak_kv_exchange_fsm.8.96623222>},infinity]}},[{gen_fsm,terminate,7,[{file,"gen_fsm.erl"},{line,611}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,227}]}]} > ancestors: [riak_kv_entropy_manager,riak_kv_sup,<0.329.0>] > messages: [{'DOWN',#Ref<0.0.36.156489>,process,<0.5596.32>,normal}] > links: [] > dictionary: [] > trap_exit: false > status: running > heap_size: 610 > stack_size: 24 > reductions: 271 > neighbours: > > _______________________________________________ > riak-users mailing list > [email protected] > http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com _______________________________________________ riak-users mailing list [email protected] http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com
