Hi, I am running on a 9 node cassandra cluster (cassandra version 3.11.5 , OS: centos 7.6). We have been running this cluster from long time, but now for a couple of days we are receiving strange errors from the same specific node again and again, during the error i don't find anything unusual on the node. below are the error from application side :
java.util.concurrent.CompletionException: java.io.IOException: Failed to open native connection to Cassandra at {xx.xx.xx.xx}:9042 at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:273) at java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:280) at java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1592) at java.util.concurrent.CompletableFuture$AsyncSupply.exec(CompletableFuture.java:1582) at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056) at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692) at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:157) Caused by: java.io.IOException: Failed to open native connection to Cassandra at {xx.xx.xx.xx}:9042 at com.datastax.spark.connector.cql.CassandraConnector$.com$datastax$spark$connector$cql$CassandraConnector$$createSession(CassandraConnector.scala:168) at com.datastax.spark.connector.cql.CassandraConnector$$anonfun$8.apply(CassandraConnector.scala:154) at com.datastax.spark.connector.cql.CassandraConnector$$anonfun$8.apply(CassandraConnector.scala:154) at com.datastax.spark.connector.cql.RefCountedCache.createNewValueAndKeys(RefCountedCache.scala:32) at com.datastax.spark.connector.cql.RefCountedCache.syncAcquire(RefCountedCache.scala:69) at com.datastax.spark.connector.cql.RefCountedCache.acquire(RefCountedCache.scala:57) at com.datastax.spark.connector.cql.CassandraConnector.openSession(CassandraConnector.scala:79) at com.datastax.spark.connector.cql.CassandraConnector.withSessionDo(CassandraConnector.scala:111) at com.datastax.spark.connector.cql.CassandraConnector.withClusterDo(CassandraConnector.scala:122) at com.datastax.spark.connector.cql.Schema$.fromCassandra(Schema.scala:332) at com.datastax.spark.connector.cql.Schema$.tableFromCassandra(Schema.scala:352) at com.datastax.spark.connector.writer.ReplicaLocator$.apply(ReplicaLocator.scala:62) at com.datastax.spark.connector.RDDFunctions.repartitionByCassandraReplica(RDDFunctions.scala:238) at com.datastax.spark.connector.japi.RDDJavaFunctions.repartitionByCassandraReplica(RDDJavaFunctions.java:157) at jobs.GenerateContestSplitPDF.getLutvDataSet(GenerateContestSplitPDF.java:119) at jobs.GenerateContestSplitPDF.lambda$main$0(GenerateContestSplitPDF.java:66) at java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1590) ... 5 more Caused by: com.datastax.driver.core.exceptions.NoHostAvailableException: All host(s) tried for query failed (tried: cps-cassandra/xx.xx.xx.xx:9042 (com.datastax.driver.core.exceptions.TransportException: [cps-cassandra/xx.xx.xx.xx:9042] Cannot connect)) at com.datastax.driver.core.ControlConnection.reconnectInternal(ControlConnection.java:233) at com.datastax.driver.core.ControlConnection.connect(ControlConnection.java:79) at com.datastax.driver.core.Cluster$Manager.init(Cluster.java:1483) at com.datastax.driver.core.Cluster.getMetadata(Cluster.java:399) at com.datastax.spark.connector.cql.CassandraConnector$.com$datastax$spark$connector$cql$CassandraConnector$$createSession(CassandraConnector.scala:161) ... 21 more java.lang.Exception: java.util.concurrent.CompletionException: java.io.IOException: Failed to open native connection to Cassandra at {xx.xx.xx.xx}:9042 at xx.xx.utils.Utils.handleException(Utils.java:123) at jobs.GenerateContestSplitPDF.lambda$main$3(GenerateContestSplitPDF.java:77) at java.util.concurrent.CompletableFuture.uniExceptionally(CompletableFuture.java:870) at java.util.concurrent.CompletableFuture$UniExceptionally.tryFire(CompletableFuture.java:852) at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474) at java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1595) at java.util.concurrent.CompletableFuture$AsyncSupply.exec(CompletableFuture.java:1582) at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056) at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692) at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:157) Caused by: java.util.concurrent.CompletionException: java.io.IOException: Failed to open native connection to Cassandra at {xx.xx.xx.xx}:9042 at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:273) at java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:280) at java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1592) ... 5 more Caused by: java.io.IOException: Failed to open native connection to Cassandra at {xx.xx.xx.xx}:9042 at com.datastax.spark.connector.cql.CassandraConnector$.com$datastax$spark$connector$cql$CassandraConnector$$createSession(CassandraConnector.scala:168) at com.datastax.spark.connector.cql.CassandraConnector$$anonfun$8.apply(CassandraConnector.scala:154) at com.datastax.spark.connector.cql.CassandraConnector$$anonfun$8.apply(CassandraConnector.scala:154) at com.datastax.spark.connector.cql.RefCountedCache.createNewValueAndKeys(RefCountedCache.scala:32) at com.datastax.spark.connector.cql.RefCountedCache.syncAcquire(RefCountedCache.scala:69) at com.datastax.spark.connector.cql.RefCountedCache.acquire(RefCountedCache.scala:57) at com.datastax.spark.connector.cql.CassandraConnector.openSession(CassandraConnector.scala:79) at com.datastax.spark.connector.cql.CassandraConnector.withSessionDo(CassandraConnector.scala:111) at com.datastax.spark.connector.cql.CassandraConnector.withClusterDo(CassandraConnector.scala:122) at com.datastax.spark.connector.cql.Schema$.fromCassandra(Schema.scala:332) at com.datastax.spark.connector.cql.Schema$.tableFromCassandra(Schema.scala:352) at com.datastax.spark.connector.writer.ReplicaLocator$.apply(ReplicaLocator.scala:62) at com.datastax.spark.connector.RDDFunctions.repartitionByCassandraReplica(RDDFunctions.scala:238) at com.datastax.spark.connector.japi.RDDJavaFunctions.repartitionByCassandraReplica(RDDJavaFunctions.java:157) at jobs.GenerateContestSplitPDF.getLutvDataSet(GenerateContestSplitPDF.java:119) at jobs.GenerateContestSplitPDF.lambda$main$0(GenerateContestSplitPDF.java:66) at java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1590) ... 5 more Caused by: com.datastax.driver.core.exceptions.NoHostAvailableException: All host(s) tried for query failed (tried: ip-xx-xx-xx-xx.ec2.internal/xx.xx.xx.xx:9042 (com.datastax.driver.core.exceptions.TransportException: [ip-xx-xx-xx-xx.ec2.internal/xx.xx.xx.xx:9042] Cannot connect)) at com.datastax.driver.core.ControlConnection.reconnectInternal(ControlConnection.java:233) at com.datastax.driver.core.ControlConnection.connect(ControlConnection.java:79) at com.datastax.driver.core.Cluster$Manager.init(Cluster.java:1483) at com.datastax.driver.core.Cluster.getMetadata(Cluster.java:399) at com.datastax.spark.connector.cql.CassandraConnector$.com$datastax$spark$connector$cql$CassandraConnector$$createSession(CassandraConnector.scala:161) ... 21 more 2020-12-18 07:06:31 WARN Session:378 - Error creating pool to ip-xx-xx-xx-xx.ec2.internal/xx.xx.xx.xx:9042 com.datastax.driver.core.exceptions.ConnectionException: [ip-xx-xx-xx-xx.ec2.internal/xx.xx.xx.xx:9042] Pool was closed during initialization at com.datastax.driver.core.HostConnectionPool$2.onSuccess(HostConnectionPool.java:148) at com.datastax.driver.core.HostConnectionPool$2.onSuccess(HostConnectionPool.java:134) at shade.com.datastax.spark.connector.google.common.util.concurrent.Futures$6.run(Futures.java:1319) at shade.com.datastax.spark.connector.google.common.util.concurrent.MoreExecutors$DirectExecutorService.execute(MoreExecutors.java:299) at shade.com.datastax.spark.connector.google.common.util.concurrent.ExecutionList.executeListener(ExecutionList.java:156) at shade.com.datastax.spark.connector.google.common.util.concurrent.ExecutionList.execute(ExecutionList.java:145) at shade.com.datastax.spark.connector.google.common.util.concurrent.AbstractFuture.set(AbstractFuture.java:185) at shade.com.datastax.spark.connector.google.common.util.concurrent.Futures$CombinedFuture.setOneValue(Futures.java:1764) at shade.com.datastax.spark.connector.google.common.util.concurrent.Futures$CombinedFuture.access$400(Futures.java:1608) at shade.com.datastax.spark.connector.google.common.util.concurrent.Futures$CombinedFuture$2.run(Futures.java:1686) at shade.com.datastax.spark.connector.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:457) at shade.com.datastax.spark.connector.google.common.util.concurrent.ExecutionList.executeListener(ExecutionList.java:156) at shade.com.datastax.spark.connector.google.common.util.concurrent.ExecutionList.execute(ExecutionList.java:145) at shade.com.datastax.spark.connector.google.common.util.concurrent.AbstractFuture.set(AbstractFuture.java:185) at shade.com.datastax.spark.connector.google.common.util.concurrent.Futures$FallbackFuture$1$1.onSuccess(Futures.java:479) at shade.com.datastax.spark.connector.google.common.util.concurrent.Futures$6.run(Futures.java:1319) at shade.com.datastax.spark.connector.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:457) at shade.com.datastax.spark.connector.google.common.util.concurrent.Futures$ImmediateFuture.addListener(Futures.java:106) at shade.com.datastax.spark.connector.google.common.util.concurrent.Futures.addCallback(Futures.java:1322) at shade.com.datastax.spark.connector.google.common.util.concurrent.Futures$FallbackFuture$1.onFailure(Futures.java:476) at shade.com.datastax.spark.connector.google.common.util.concurrent.Futures$6.run(Futures.java:1310) at shade.com.datastax.spark.connector.google.common.util.concurrent.MoreExecutors$DirectExecutorService.execute(MoreExecutors.java:299) at shade.com.datastax.spark.connector.google.common.util.concurrent.ExecutionList.executeListener(ExecutionList.java:156) at shade.com.datastax.spark.connector.google.common.util.concurrent.ExecutionList.execute(ExecutionList.java:145) at shade.com.datastax.spark.connector.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:202) at shade.com.datastax.spark.connector.google.common.util.concurrent.Futures$FallbackFuture$1$1.onFailure(Futures.java:487) at shade.com.datastax.spark.connector.google.common.util.concurrent.Futures$6.run(Futures.java:1310) at shade.com.datastax.spark.connector.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:457) at shade.com.datastax.spark.connector.google.common.util.concurrent.ExecutionList.executeListener(ExecutionList.java:156) at shade.com.datastax.spark.connector.google.common.util.concurrent.ExecutionList.add(ExecutionList.java:101) at shade.com.datastax.spark.connector.google.common.util.concurrent.AbstractFuture.addListener(AbstractFuture.java:170) at shade.com.datastax.spark.connector.google.common.util.concurrent.Futures.addCallback(Futures.java:1322) at shade.com.datastax.spark.connector.google.common.util.concurrent.Futures$FallbackFuture$1.onFailure(Futures.java:476) at shade.com.datastax.spark.connector.google.common.util.concurrent.Futures$6.run(Futures.java:1310) at shade.com.datastax.spark.connector.google.common.util.concurrent.MoreExecutors$DirectExecutorService.execute(MoreExecutors.java:299) at shade.com.datastax.spark.connector.google.common.util.concurrent.ExecutionList.executeListener(ExecutionList.java:156) at shade.com.datastax.spark.connector.google.common.util.concurrent.ExecutionList.execute(ExecutionList.java:145) at shade.com.datastax.spark.connector.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:202) at shade.com.datastax.spark.connector.google.common.util.concurrent.Futures$ChainingListenableFuture.run(Futures.java:902) at shade.com.datastax.spark.connector.google.common.util.concurrent.Futures$1$1.run(Futures.java:635) at shade.com.datastax.spark.connector.google.common.util.concurrent.MoreExecutors$DirectExecutorService.execute(MoreExecutors.java:299) at shade.com.datastax.spark.connector.google.common.util.concurrent.Futures$1.run(Futures.java:632) at shade.com.datastax.spark.connector.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:457) at shade.com.datastax.spark.connector.google.common.util.concurrent.ExecutionList.executeListener(ExecutionList.java:156) at shade.com.datastax.spark.connector.google.common.util.concurrent.ExecutionList.execute(ExecutionList.java:145) at shade.com.datastax.spark.connector.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:202) at shade.com.datastax.spark.connector.google.common.util.concurrent.SettableFuture.setException(SettableFuture.java:68) at com.datastax.driver.core.Connection$1.operationComplete(Connection.java:166) at com.datastax.driver.core.Connection$1.operationComplete(Connection.java:149) at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:507) at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:500) at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:479) at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:420) at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:122) at io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:659) at io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:678) at io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:552) at io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:394) at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:304) at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:138) at java.lang.Thread.run(Thread.java:748) Any suggestions ? -- regards Vikash Rajoriya +91-8904505536