Ambari metrics collector was working fine for 6 months and suddenly it
stopped working. This is the error we are getting.
hbase.rootdir - /mnt/data/ambari-metrics-collector/hbase
hbase.cluster.distributed -false
Metrics service operation mode -embedded
hbase.zookeeper.property.clientPort - 61181
06:19:30,678 WARN [main] RecoverableZooKeeper:253 - Possibly transient
ZooKeeper, quorum=localhost:61181,
exception=org.apache.zookeeper.KeeperException$ConnectionLossException:
KeeperErrorCode = Connect ionLoss for /hbase 06:19:30,678 ERROR [main]
RecoverableZooKeeper:255 - ZooKeeper exists failed after 4 attempts
06:19:30,679 WARN [main] ZKUtil:484 - hconnection-0xd78795,
quorum=localhost:61181, baseZNode=/hbase Unable to set watcher on znode
(/hbase) org.apache.zookeeper.KeeperException$ConnectionLossException:
KeeperErrorCode = ConnectionLoss for /hbase at
org.apache.zookeeper.KeeperException.create(KeeperException.java:99) at
org.apache.zookeeper.KeeperException.create(KeeperException.java:51) at
org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1045) at
org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:199)
at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:481) at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.checkIfBaseNodeAvailable(ConnectionManager.java:874)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.access$600(ConnectionManager.java:585)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation$StubMaker.makeStubNoRetries(ConnectionManager.java:1553)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation$StubMaker.makeStub(ConnectionManager.java:1599)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation$MasterServiceStubMaker.makeStub(ConnectionManager.java:1653)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.getKeepAliveMasterService(ConnectionManager.java:1860)
at
org.apache.hadoop.hbase.client.HBaseAdmin$MasterCallable.prepare(HBaseAdmin.java:3363)
at
org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:125)
at
org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3390)
at
org.apache.hadoop.hbase.client.HBaseAdmin.getTableDescriptor(HBaseAdmin.java:408)
at
org.apache.hadoop.hbase.client.HBaseAdmin.getTableDescriptor(HBaseAdmin.java:429)
at
org.apache.phoenix.query.ConnectionQueryServicesImpl.ensureTableCreated(ConnectionQueryServicesImpl.java:762)
at
org.apache.phoenix.query.ConnectionQueryServicesImpl.createTable(ConnectionQueryServicesImpl.java:1107)
at
org.apache.phoenix.query.DelegateConnectionQueryServices.createTable(DelegateConnectionQueryServices.java:110)
at
org.apache.phoenix.schema.MetaDataClient.createTableInternal(MetaDataClient.java:1527)
at
org.apache.phoenix.schema.MetaDataClient.createTable(MetaDataClient.java:535)
at
org.apache.phoenix.compile.CreateTableCompiler$2.execute(CreateTableCompiler.java:184)
at
org.apache.phoenix.jdbc.PhoenixStatement$2.call(PhoenixStatement.java:260)
at
org.apache.phoenix.jdbc.PhoenixStatement$2.call(PhoenixStatement.java:252)
at org.apache.phoenix.call.CallRunner.run(CallRunner.java:53) at
org.apache.phoenix.jdbc.PhoenixStatement.executeMutation(PhoenixStatement.java:250)
at
org.apache.phoenix.jdbc.PhoenixStatement.executeUpdate(PhoenixStatement.java:1026)
at
org.apache.phoenix.query.ConnectionQueryServicesImpl$9.call(ConnectionQueryServicesImpl.java:1532)
06:19:30,678 WARN [main] RecoverableZooKeeper:253 - Possibly transient
ZooKeeper, quorum=localhost:61181,
exception=org.apache.zookeeper.KeeperException$ConnectionLossException:
KeeperErrorCode = Connect ionLoss for /hbase 06:19:30,678 ERROR [main]
RecoverableZooKeeper:255 - ZooKeeper exists failed after 4 attempts
06:19:30,679 WARN [main] ZKUtil:484 - hconnection-0xd78795,
quorum=localhost:61181, baseZNode=/hbase Unable to set watcher on znode
(/hbase) org.apache.zookeeper.KeeperException$ConnectionLossException:
KeeperErrorCode = ConnectionLoss for /hbase at
org.apache.zookeeper.KeeperException.create(KeeperException.java:99) at
org.apache.zookeeper.KeeperException.create(KeeperException.java:51) at
org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1045) at
org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:199)
at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:481) at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.checkIfBaseNodeAvailable(ConnectionManager.java:874)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.access$600(ConnectionManager.java:585)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation$StubMaker.makeStubNoRetries(ConnectionManager.java:1553)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation$StubMaker.makeStub(ConnectionManager.java:1599)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation$MasterServiceStubMaker.makeStub(ConnectionManager.java:1653)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.getKeepAliveMasterService(ConnectionManager.java:1860)
at
org.apache.hadoop.hbase.client.HBaseAdmin$MasterCallable.prepare(HBaseAdmin.java:3363)
at
org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:125)
at
org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3390)
at
org.apache.hadoop.hbase.client.HBaseAdmin.getTableDescriptor(HBaseAdmin.java:408)
at
org.apache.hadoop.hbase.client.HBaseAdmin.getTableDescriptor(HBaseAdmin.java:429)
at
org.apache.phoenix.query.ConnectionQueryServicesImpl.ensureTableCreated(ConnectionQueryServicesImpl.java:762)
at
org.apache.phoenix.query.ConnectionQueryServicesImpl.createTable(ConnectionQueryServicesImpl.java:1107)
at
org.apache.phoenix.query.DelegateConnectionQueryServices.createTable(DelegateConnectionQueryServices.java:110)
at
org.apache.phoenix.schema.MetaDataClient.createTableInternal(MetaDataClient.java:1527)
at
org.apache.phoenix.schema.MetaDataClient.createTable(MetaDataClient.java:535)
at
org.apache.phoenix.compile.CreateTableCompiler$2.execute(CreateTableCompiler.java:184)
at
org.apache.phoenix.jdbc.PhoenixStatement$2.call(PhoenixStatement.java:260)
at
org.apache.phoenix.jdbc.PhoenixStatement$2.call(PhoenixStatement.java:252)
at org.apache.phoenix.call.CallRunner.run(CallRunner.java:53) at
org.apache.phoenix.jdbc.PhoenixStatement.executeMutation(PhoenixStatement.java:250)
at
org.apache.phoenix.jdbc.PhoenixStatement.executeUpdate(PhoenixStatement.java:1026)
at
org.apache.phoenix.query.ConnectionQueryServicesImpl$9.call(ConnectionQueryServicesImpl.java:1532)
On Sat, May 7, 2016 at 12:37 AM, Anandha L Ranganathan <
[email protected]> wrote:
> Ambari metrics collector was working fine for 6 months and suddenly it
> stopped working. This is the error we are getting.
>
> hbase.rootdir - /mnt/data/ambari-metrics-collector/hbase
> hbase.cluster.distributed -false
> Metrics service operation mode -embedded
> hbase.zookeeper.property.clientPort - 61181
> quorum =localhost
>
>
>
>
> - 06:19:30,678 WARN [main] RecoverableZooKeeper:253 - Possibly
> transient ZooKeeper, quorum=localhost:61181,
> exception=org.apache.zookeeper.KeeperException$ConnectionLossException:
> KeeperErrorCode = Connect ionLoss for /hbase 06:19:30,678 ERROR [main]
> RecoverableZooKeeper:255 - ZooKeeper exists failed after 4 attempts
> 06:19:30,679 WARN [main] ZKUtil:484 - hconnection-0xd78795,
> quorum=localhost:61181, baseZNode=/hbase Unable to set watcher on znode
> (/hbase) org.apache.zookeeper.KeeperException$ConnectionLossException:
> KeeperErrorCode = ConnectionLoss for /hbase at
> org.apache.zookeeper.KeeperException.create(KeeperException.java:99) at
> org.apache.zookeeper.KeeperException.create(KeeperException.java:51) at
> org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1045) at
>
> org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:199)
> at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:481) at
>
> org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.checkIfBaseNodeAvailable(ConnectionManager.java:874)
> at
>
> org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.access$600(ConnectionManager.java:585)
> at
>
> org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation$StubMaker.makeStubNoRetries(ConnectionManager.java:1553)
> at
>
> org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation$StubMaker.makeStub(ConnectionManager.java:1599)
> at
>
> org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation$MasterServiceStubMaker.makeStub(ConnectionManager.java:1653)
> at
>
> org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.getKeepAliveMasterService(ConnectionManager.java:1860)
> at
>
> org.apache.hadoop.hbase.client.HBaseAdmin$MasterCallable.prepare(HBaseAdmin.java:3363)
> at
>
> org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:125)
> at
>
> org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3390)
> at
>
> org.apache.hadoop.hbase.client.HBaseAdmin.getTableDescriptor(HBaseAdmin.java:408)
> at
>
> org.apache.hadoop.hbase.client.HBaseAdmin.getTableDescriptor(HBaseAdmin.java:429)
> at
>
> org.apache.phoenix.query.ConnectionQueryServicesImpl.ensureTableCreated(ConnectionQueryServicesImpl.java:762)
> at
>
> org.apache.phoenix.query.ConnectionQueryServicesImpl.createTable(ConnectionQueryServicesImpl.java:1107)
> at
>
> org.apache.phoenix.query.DelegateConnectionQueryServices.createTable(DelegateConnectionQueryServices.java:110)
> at
>
> org.apache.phoenix.schema.MetaDataClient.createTableInternal(MetaDataClient.java:1527)
> at
>
> org.apache.phoenix.schema.MetaDataClient.createTable(MetaDataClient.java:535)
> at
>
> org.apache.phoenix.compile.CreateTableCompiler$2.execute(CreateTableCompiler.java:184)
> at
> org.apache.phoenix.jdbc.PhoenixStatement$2.call(PhoenixStatement.java:260)
> at
> org.apache.phoenix.jdbc.PhoenixStatement$2.call(PhoenixStatement.java:252)
> at org.apache.phoenix.call.CallRunner.run(CallRunner.java:53) at
>
> org.apache.phoenix.jdbc.PhoenixStatement.executeMutation(PhoenixStatement.java:250)
> at
>
> org.apache.phoenix.jdbc.PhoenixStatement.executeUpdate(PhoenixStatement.java:1026)
> at
>
> org.apache.phoenix.query.ConnectionQueryServicesImpl$9.call(ConnectionQueryServicesImpl.java:1532)
>
>
>
>
