Hello, all
It's my understanding that it's best practice to colocate thrift on each of
the regionservers, and use load balancing to spread queries between them.
However, I can't seem to get thrift to bind from the regionservers (only
from the hmaster).
The machines have connectivity between them; regionservers connect and
communicate with the hmaster without any problems.
I'm running Hadoop 0.20.2-cdh3u2, HBase 0.90.4-cdh3u2.
Thrift can run succesfully from the HMaster, but not from any of the
datanode/regionservers
What appears to be the relevant content from
hbase-hbase-thrift-region-1.out below:
12/02/09 19:09:17 INFO client.HConnectionManager$HConnectionImplementation:
getMaster attempt 0 of 10 failed; retrying after sleep of 1000
java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
at
org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
at
org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
at
org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
at
org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
at
org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
at $Proxy4.getProtocolVersion(Unknown Source)
at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
at
org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
at
org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
at
org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
12/02/09 19:09:18 INFO client.HConnectionManager$HConnectionImplementation:
getMaster attempt 1 of 10 failed; retrying after sleep of 1000
java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
at
org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
at
org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
at
org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
at
org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
at
org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
at $Proxy4.getProtocolVersion(Unknown Source)
at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
at
org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
at
org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
at
org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
12/02/09 19:09:19 INFO client.HConnectionManager$HConnectionImplementation:
getMaster attempt 2 of 10 failed; retrying after sleep of 1000
java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
at
org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
at
org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
at
org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
at
org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
at
org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
at $Proxy4.getProtocolVersion(Unknown Source)
at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
at
org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
at
org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
at
org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
12/02/09 19:09:20 INFO client.HConnectionManager$HConnectionImplementation:
getMaster attempt 3 of 10 failed; retrying after sleep of 2000
java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
at
org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
at
org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
at
org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
at
org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
at
org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
at $Proxy4.getProtocolVersion(Unknown Source)
at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
at
org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
at
org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
at
org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
12/02/09 19:09:22 INFO client.HConnectionManager$HConnectionImplementation:
getMaster attempt 4 of 10 failed; retrying after sleep of 2000
java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
at
org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
at
org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
at
org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
at
org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
at
org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
at $Proxy4.getProtocolVersion(Unknown Source)
at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
at
org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
at
org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
at
org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
12/02/09 19:09:24 INFO client.HConnectionManager$HConnectionImplementation:
getMaster attempt 5 of 10 failed; retrying after sleep of 4000
java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
at
org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
at
org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
at
org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
at
org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
at
org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
at $Proxy4.getProtocolVersion(Unknown Source)
at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
at
org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
at
org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
at
org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
12/02/09 19:09:28 INFO client.HConnectionManager$HConnectionImplementation:
getMaster attempt 6 of 10 failed; retrying after sleep of 4000
java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
at
org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
at
org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
at
org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
at
org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
at
org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
at $Proxy4.getProtocolVersion(Unknown Source)
at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
at
org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
at
org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
at
org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
12/02/09 19:09:32 INFO client.HConnectionManager$HConnectionImplementation:
getMaster attempt 7 of 10 failed; retrying after sleep of 8000
java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
at
org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
at
org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
at
org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
at
org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
at
org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
at $Proxy4.getProtocolVersion(Unknown Source)
at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
at
org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
at
org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
at
org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
12/02/09 19:09:40 INFO client.HConnectionManager$HConnectionImplementation:
getMaster attempt 8 of 10 failed; retrying after sleep of 16000
java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
at
org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
at
org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
at
org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
at
org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
at
org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
at $Proxy4.getProtocolVersion(Unknown Source)
at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
at
org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
at
org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
at
org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
12/02/09 19:09:56 INFO client.HConnectionManager$HConnectionImplementation:
getMaster attempt 9 of 10 failed; no more retrying.
java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
at
org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
at
org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
at
org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
at
org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
at
org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
at $Proxy4.getProtocolVersion(Unknown Source)
at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
at
org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
at
org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
at
org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
Exception in thread "main"
org.apache.hadoop.hbase.MasterNotRunningException: master:60000
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:396)
at
org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
at
org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
at
org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
