Are the client and the server both framed or not framed?
On Wed, Mar 27, 2013 at 7:31 PM, Mingqi Shao <[email protected]> wrote: > Hi every, > > I developed a thrift server which implement scribe interface. For some > reason, my thrift server was stuck there. symptom is : > 1. Client can connect to server, but not response till timeout > 2. the client's request data can't get to server's handler. But no any > exception or error in server side. > 3. the output of server's netstat is: > tcp 0 0 0.0.0.0:1588 0.0.0.0:* > LISTEN > tcp 0 0 192.168.4.108:1588 192.168.1.192:43934 > SYN_RECV > tcp 0 0 192.168.4.108:1588 192.168.1.192:43932 > SYN_RECV > tcp 0 0 192.168.4.108:1588 172.16.10.225:48950 > ESTABLISHED > tcp 0 0 192.168.4.108:1588 172.16.10.225:45488 > ESTABLISHED > tcp 66320 0 192.168.4.108:1588 172.16.10.225:47272 > ESTABLISHED > tcp 0 0 192.168.4.108:1588 172.16.10.225:48550 > ESTABLISHED > > 4. I juse jstack to look into JVM, output here: > > 2013-03-27 22:13:33 > Full thread dump Java HotSpot(TM) 64-Bit Server VM (10.0-b22 mixed mode): > > "Attach Listener" daemon prio=10 tid=0x000000004e9e5000 nid=0x35f2 waiting > on condition [0x0000000000000000..0x0000000000000000] > java.lang.Thread.State: RUNNABLE > > "pool-1-thread-9" prio=10 tid=0x000000004ea50800 nid=0x7227 runnable > [0x00000000420e7000..0x00000000420e7a10] > java.lang.Thread.State: RUNNABLE > at java.net.SocketInputStream.socketRead0(Native Method) > at java.net.SocketInputStream.read(SocketInputStream.java:129) > at java.io.BufferedInputStream.fill(BufferedInputStream.java:218) > at java.io.BufferedInputStream.read1(BufferedInputStream.java:258) > at java.io.BufferedInputStream.read(BufferedInputStream.java:317) > - locked <0x00002aaab37083c0> (a java.io.BufferedInputStream) > at > > org.apache.thrift.transport.TIOStreamTransport.read(TIOStreamTransport.java:127) > at > org.apache.thrift.transport.TTransport.readAll(TTransport.java:84) > at > > org.apache.thrift.transport.TFramedTransport.readFrame(TFramedTransport.java:129) > at > > org.apache.thrift.transport.TFramedTransport.read(TFramedTransport.java:101) > at > org.apache.thrift.transport.TTransport.readAll(TTransport.java:84) > at > > org.apache.thrift.protocol.TBinaryProtocol.readAll(TBinaryProtocol.java:378) > at > > org.apache.thrift.protocol.TBinaryProtocol.readI32(TBinaryProtocol.java:297) > at > > org.apache.thrift.protocol.TBinaryProtocol.readMessageBegin(TBinaryProtocol.java:204) > at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:27) > at > > org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:206) > at > > java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:885) > at > > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) > at java.lang.Thread.run(Thread.java:619) > > "pool-1-thread-8" prio=10 tid=0x000000004e97f400 nid=0x5304 runnable > [0x00000000425ec000..0x00000000425ecb90] > java.lang.Thread.State: RUNNABLE > at java.net.SocketInputStream.socketRead0(Native Method) > at java.net.SocketInputStream.read(SocketInputStream.java:129) > at java.io.BufferedInputStream.fill(BufferedInputStream.java:218) > at java.io.BufferedInputStream.read1(BufferedInputStream.java:258) > at java.io.BufferedInputStream.read(BufferedInputStream.java:317) > - locked <0x00002aaab37044c0> (a java.io.BufferedInputStream) > at > > org.apache.thrift.transport.TIOStreamTransport.read(TIOStreamTransport.java:127) > at > org.apache.thrift.transport.TTransport.readAll(TTransport.java:84) > at > > org.apache.thrift.transport.TFramedTransport.readFrame(TFramedTransport.java:129) > at > > org.apache.thrift.transport.TFramedTransport.read(TFramedTransport.java:101) > at > org.apache.thrift.transport.TTransport.readAll(TTransport.java:84) > at > > org.apache.thrift.protocol.TBinaryProtocol.readAll(TBinaryProtocol.java:378) > at > > org.apache.thrift.protocol.TBinaryProtocol.readI32(TBinaryProtocol.java:297) > at > > org.apache.thrift.protocol.TBinaryProtocol.readMessageBegin(TBinaryProtocol.java:204) > at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:27) > at > > org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:206) > at > > java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:885) > at > > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) > at java.lang.Thread.run(Thread.java:619) > "pool-1-thread-1" prio=10 tid=0x000000004e9ba800 nid=0x3da6 runnable > [0x00000000415fb000..0x00000000415fbc10] > java.lang.Thread.State: RUNNABLE > at java.net.SocketInputStream.socketRead0(Native Method) > at java.net.SocketInputStream.read(SocketInputStream.java:129) > at java.io.BufferedInputStream.fill(BufferedInputStream.java:218) > at java.io.BufferedInputStream.read1(BufferedInputStream.java:258) > at java.io.BufferedInputStream.read(BufferedInputStream.java:317) > - locked <0x00002aaab3642e40> (a java.io.BufferedInputStream) > at > > org.apache.thrift.transport.TIOStreamTransport.read(TIOStreamTransport.java:127) > at > org.apache.thrift.transport.TTransport.readAll(TTransport.java:84) > at > > org.apache.thrift.transport.TFramedTransport.readFrame(TFramedTransport.java:129) > at > > org.apache.thrift.transport.TFramedTransport.read(TFramedTransport.java:101) > at > org.apache.thrift.transport.TTransport.readAll(TTransport.java:84) > at > > org.apache.thrift.protocol.TBinaryProtocol.readAll(TBinaryProtocol.java:378) > at > > org.apache.thrift.protocol.TBinaryProtocol.readI32(TBinaryProtocol.java:297) > at > > org.apache.thrift.protocol.TBinaryProtocol.readMessageBegin(TBinaryProtocol.java:204) > at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:27) > at > > org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:206) > at > > java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:885) > at > > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) > at java.lang.Thread.run(Thread.java:619) > > "DestroyJavaVM" prio=10 tid=0x000000004e4e1000 nid=0x3d79 waiting on > condition [0x0000000000000000..0x00000000404cdd00] > java.lang.Thread.State: RUNNABLE > > "Low Memory Detector" daemon prio=10 tid=0x000000004e599c00 nid=0x3d82 > runnable [0x0000000000000000..0x0000000000000000] > java.lang.Thread.State: RUNNABLE > > "CompilerThread1" daemon prio=10 tid=0x000000004e597800 nid=0x3d81 waiting > on condition [0x0000000000000000..0x0000000041be12b0] > java.lang.Thread.State: RUNNABLE > > "CompilerThread0" daemon prio=10 tid=0x000000004e594000 nid=0x3d80 waiting > on condition [0x0000000000000000..0x000000004186a330] > java.lang.Thread.State: RUNNABLE > > "Signal Dispatcher" daemon prio=10 tid=0x000000004e592800 nid=0x3d7f > runnable [0x0000000000000000..0x000000004176a7a0] > java.lang.Thread.State: RUNNABLE > > "Finalizer" daemon prio=10 tid=0x000000004e567000 nid=0x3d7e in > Object.wait() [0x0000000041ae1000..0x0000000041ae1b90] > java.lang.Thread.State: WAITING (on object monitor) > at java.lang.Object.wait(Native Method) > - waiting on <0x00002aaab36831f8> (a > java.lang.ref.ReferenceQueue$Lock) > at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:116) > - locked <0x00002aaab36831f8> (a java.lang.ref.ReferenceQueue$Lock) > at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:132) > at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159) > > "Reference Handler" daemon prio=10 tid=0x000000004e565c00 nid=0x3d7d in > Object.wait() [0x00000000419e0000..0x00000000419e0c10] > java.lang.Thread.State: WAITING (on object monitor) > at java.lang.Object.wait(Native Method) > - waiting on <0x00002aaab3603f38> (a java.lang.ref.Reference$Lock) > at java.lang.Object.wait(Object.java:485) > at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116) > - locked <0x00002aaab3603f38> (a java.lang.ref.Reference$Lock) > > "VM Thread" prio=10 tid=0x000000004e560800 nid=0x3d7c runnable > > "GC task thread#0 (ParallelGC)" prio=10 tid=0x000000004e4eb400 nid=0x3d7a > runnable > > "GC task thread#1 (ParallelGC)" prio=10 tid=0x000000004e4ec800 nid=0x3d7b > runnable > > "VM Periodic Task Thread" prio=10 tid=0x000000004e59b800 nid=0x3d83 waiting > on condition > > JNI global references: 682 > -- Dan Di Spaltro
