Re: Cassandra restart

Wed, 14 Oct 2009 07:26:25 -0700

Sorry, I don't see anything connected to key 00000000000013062900 or message 2691, debug is turned off on that node because 10.3.2.36 wasn't restarted 

I will try to reproduce problem on smaller test cluster.

Fragment of 10.3.2.36 log (time is in sync):
ERROR [pool-1-thread-15051] 2009-10-14 18:03:30,562 StorageProxy.java (line 179) error writing key 00000000000176715312 java.util.concurrent.TimeoutException: Operation timed out - received only 0 responses from . at org.apache.cassandra.service.QuorumResponseHandler.get(QuorumResponseHandler.java:88) at org.apache.cassandra.service.StorageProxy.insertBlocking(StorageProxy.java:164) at org.apache.cassandra.service.CassandraServer.doInsert(CassandraServer.java:468) at org.apache.cassandra.service.CassandraServer.remove(CassandraServer.java:461) at org.apache.cassandra.service.Cassandra$Processor$remove.process(Cassandra.java:884) 
    at 
org.apache.cassandra.service.Cassandra$Processor.process(Cassandra.java:627)
at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:253) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) 
    at java.lang.Thread.run(Thread.java:636)
ERROR [pool-1-thread-14947] 2009-10-14 18:03:32,593 StorageProxy.java (line 179) error writing key 00000000000176715312 java.util.concurrent.TimeoutException: Operation timed out - received only 0 responses from . at org.apache.cassandra.service.QuorumResponseHandler.get(QuorumResponseHandler.java:88) at org.apache.cassandra.service.StorageProxy.insertBlocking(StorageProxy.java:164) at org.apache.cassandra.service.CassandraServer.doInsert(CassandraServer.java:468) at org.apache.cassandra.service.CassandraServer.remove(CassandraServer.java:461) at org.apache.cassandra.service.Cassandra$Processor$remove.process(Cassandra.java:884) 
    at 
org.apache.cassandra.service.Cassandra$Processor.process(Cassandra.java:627)
at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:253) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) 
    at java.lang.Thread.run(Thread.java:636)
ERROR [pool-1-thread-14985] 2009-10-14 18:03:34,571 StorageProxy.java (line 179) error writing key 00000000000176715312 java.util.concurrent.TimeoutException: Operation timed out - received only 0 responses from . at org.apache.cassandra.service.QuorumResponseHandler.get(QuorumResponseHandler.java:88) at org.apache.cassandra.service.StorageProxy.insertBlocking(StorageProxy.java:164) at org.apache.cassandra.service.CassandraServer.doInsert(CassandraServer.java:468) at org.apache.cassandra.service.CassandraServer.remove(CassandraServer.java:461) at org.apache.cassandra.service.Cassandra$Processor$remove.process(Cassandra.java:884) 
    at 
org.apache.cassandra.service.Cassandra$Processor.process(Cassandra.java:627)
at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:253) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) 
    at java.lang.Thread.run(Thread.java:636)
ERROR [pool-1-thread-14493] 2009-10-14 18:03:34,571 Cassandra.java (line 679) Internal error processing get_slice 
java.lang.RuntimeException: error reading key 00000000000106371503
at org.apache.cassandra.service.StorageProxy.weakReadRemote(StorageProxy.java:265) at org.apache.cassandra.service.StorageProxy.readProtocol(StorageProxy.java:312) at org.apache.cassandra.service.CassandraServer.readColumnFamily(CassandraServer.java:100) at org.apache.cassandra.service.CassandraServer.getSlice(CassandraServer.java:182) at org.apache.cassandra.service.CassandraServer.multigetSliceInternal(CassandraServer.java:251) at org.apache.cassandra.service.CassandraServer.get_slice(CassandraServer.java:220) at org.apache.cassandra.service.Cassandra$Processor$get_slice.process(Cassandra.java:671) 
    at 
org.apache.cassandra.service.Cassandra$Processor.process(Cassandra.java:627)
at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:253) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) 
    at java.lang.Thread.run(Thread.java:636)
Caused by: java.util.concurrent.TimeoutException: Operation timed out.
    at org.apache.cassandra.net.AsyncResult.get(AsyncResult.java:97)
at org.apache.cassandra.service.StorageProxy.weakReadRemote(StorageProxy.java:261) 
    ... 11 more
ERROR [pool-1-thread-14820] 2009-10-14 18:03:35,406 Cassandra.java (line 679) Internal error processing get_slice 
java.lang.RuntimeException: error reading key 00000000000019586353
at org.apache.cassandra.service.StorageProxy.weakReadRemote(StorageProxy.java:265) at org.apache.cassandra.service.StorageProxy.readProtocol(StorageProxy.java:312) at org.apache.cassandra.service.CassandraServer.readColumnFamily(CassandraServer.java:100) at org.apache.cassandra.service.CassandraServer.getSlice(CassandraServer.java:182) at org.apache.cassandra.service.CassandraServer.multigetSliceInternal(CassandraServer.java:251) at org.apache.cassandra.service.CassandraServer.get_slice(CassandraServer.java:220) at org.apache.cassandra.service.Cassandra$Processor$get_slice.process(Cassandra.java:671) 
    at 
org.apache.cassandra.service.Cassandra$Processor.process(Cassandra.java:627)
at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:253) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) 
    at java.lang.Thread.run(Thread.java:636)
Caused by: java.util.concurrent.TimeoutException: Operation timed out.
    at org.apache.cassandra.net.AsyncResult.get(AsyncResult.java:97)
at org.apache.cassandra.service.StorageProxy.weakReadRemote(StorageProxy.java:261) 
    ... 11 more
ERROR [pool-1-thread-15051] 2009-10-14 18:03:36,602 StorageProxy.java (line 179) error writing key 00000000000176715312 java.util.concurrent.TimeoutException: Operation timed out - received only 0 responses from . at org.apache.cassandra.service.QuorumResponseHandler.get(QuorumResponseHandler.java:88) at org.apache.cassandra.service.StorageProxy.insertBlocking(StorageProxy.java:164) at org.apache.cassandra.service.CassandraServer.doInsert(CassandraServer.java:468) at org.apache.cassandra.service.CassandraServer.remove(CassandraServer.java:461) at org.apache.cassandra.service.Cassandra$Processor$remove.process(Cassandra.java:884) 
    at 
org.apache.cassandra.service.Cassandra$Processor.process(Cassandra.java:627)
at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:253) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) 
    at java.lang.Thread.run(Thread.java:636)
ERROR [pool-1-thread-14658] 2009-10-14 18:03:37,590 Cassandra.java (line 679) Internal error processing get_slice 
java.lang.RuntimeException: error reading key 00000000000139616600
at org.apache.cassandra.service.StorageProxy.weakReadRemote(StorageProxy.java:265) at org.apache.cassandra.service.StorageProxy.readProtocol(StorageProxy.java:312) at org.apache.cassandra.service.CassandraServer.readColumnFamily(CassandraServer.java:100) at org.apache.cassandra.service.CassandraServer.getSlice(CassandraServer.java:182) at org.apache.cassandra.service.CassandraServer.multigetSliceInternal(CassandraServer.java:251) at org.apache.cassandra.service.CassandraServer.get_slice(CassandraServer.java:220) at org.apache.cassandra.service.Cassandra$Processor$get_slice.process(Cassandra.java:671) 
    at 
org.apache.cassandra.service.Cassandra$Processor.process(Cassandra.java:627)
at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:253) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) 
    at java.lang.Thread.run(Thread.java:636)
Caused by: java.util.concurrent.TimeoutException: Operation timed out.
    at org.apache.cassandra.net.AsyncResult.get(AsyncResult.java:97)
at org.apache.cassandra.service.StorageProxy.weakReadRemote(StorageProxy.java:261) 
    ... 11 more


--
Teodor Sigaev                                   E-mail: [email protected]
                                                   WWW: http://www.sigaev.ru/

Reply via email to