I get the below exception on my client... #1 I rebooted the cache nodes error still continued. #2 restarted the client node error went away. #3 this seems to happen every few weeks. #4 is there some sort of timeout values and retries I can put? #5 cache operations seem to block when rebooting the nodes (I have 3 nodes). Is there a way not to block?
javax.cache.CacheException: class org.apache.ignite.internal.processors.cache.CacheInvalidStateException: Failed to execute cache operation (all partition owners have left the grid, partition data has been lost) [cacheName=xxxxxx, part=273, key=16479796986] at org.apache.ignite.internal.processors.cache.GridCacheUtils.convertToCacheException(GridCacheUtils.java:1337) at org.apache.ignite.internal.processors.cache.IgniteCacheFutureImpl.convertException(IgniteCacheFutureImpl.java:62) at org.apache.ignite.internal.util.future.IgniteFutureImpl.get(IgniteFutureImpl.java:157) at com.xxxxxx.common.vertx.ext.data.impl.IgniteCacheRepository.lambda$executeAsync$394d953f$1(IgniteCacheRepository.java:59) at org.apache.ignite.internal.util.future.AsyncFutureListener$1.run(AsyncFutureListener.java:53) at com.xxxxxx.common.vertx.ext.data.impl.VertxIgniteExecutorAdapter.lambda$execute$0(VertxIgniteExecutorAdapter.java:18) at io.vertx.core.impl.ContextImpl.executeTask(ContextImpl.java:369) at io.vertx.core.impl.EventLoopContext.lambda$executeAsync$0(EventLoopContext.java:38) at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164) at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:497) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.ignite.internal.processors.cache.CacheInvalidStateException: Failed to execute cache operation (all partition owners have left the grid, partition data has been lost) [cacheName=xxxxxx, part=273, key=16479796986] at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFutureAdapter.validatePartitionOperation(GridDhtTopologyFutureAdapter.java:161) at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFutureAdapter.validateCache(GridDhtTopologyFutureAdapter.java:116) at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicSingleUpdateFuture.mapOnTopology(GridNearAtomicSingleUpdateFuture.java:417) at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicAbstractUpdateFuture.map(GridNearAtomicAbstractUpdateFuture.java:248) at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$26.apply(GridDhtAtomicCache.java:1146) at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$26.apply(GridDhtAtomicCache.java:1144) at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.asyncOp(GridDhtAtomicCache.java:761) at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.update0(GridDhtAtomicCache.java:1144) at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.putAsync0(GridDhtAtomicCache.java:641) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.putAsync(GridCacheAdapter.java:2828) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.putAsync(GridCacheAdapter.java:2809) at org.apache.ignite.internal.processors.cache.IgniteCacheProxyImpl.putAsync0(IgniteCacheProxyImpl.java:1125) at org.apache.ignite.internal.processors.cache.IgniteCacheProxyImpl.putAsync(IgniteCacheProxyImpl.java:1114) at org.apache.ignite.internal.processors.cache.GatewayProtectedCacheProxy.putAsync(GatewayProtectedCacheProxy.java:832) at com.xxxxxx.common.vertx.ext.data.impl.IgniteCacheRepository.lambda$put$0(IgniteCacheRepository.java:27) at com.xxxxxx.common.vertx.ext.data.impl.IgniteCacheRepository.executeAsync(IgniteCacheRepository.java:55) at com.xxxxxx.common.vertx.ext.data.impl.IgniteCacheRepository.put(IgniteCacheRepository.java:27) at com.xxxxxx.service.impl.xxxxxx.put(xxxxxx.java:52) at com.xxxxxx.impl.xxxxxx.moV1(xxxxxx.java:134) at com.xxxxxx.api.xxxxxx.moV1(xxxxxx.java:100) at io.vertx.ext.web.impl.RouteState.handleContext(RouteState.java:1034) at io.vertx.ext.web.impl.RoutingContextImplBase.iterateNext(RoutingContextImplBase.java:131) at io.vertx.ext.web.impl.RoutingContextImpl.next(RoutingContextImpl.java:128) at com.xxxxxx.common.vertx.ext.web.handler.impl.JsonLoggerHandlerImpl.handle(JsonLoggerHandlerImpl.java:168) at com.xxxxxx.common.vertx.ext.web.handler.impl.JsonLoggerHandlerImpl.handle(JsonLoggerHandlerImpl.java:24) at io.vertx.ext.web.impl.RouteState.handleContext(RouteState.java:1034) at io.vertx.ext.web.impl.RoutingContextImplBase.iterateNext(RoutingContextImplBase.java:95) at io.vertx.ext.web.impl.RoutingContextImpl.next(RoutingContextImpl.java:128) at com.xxxxxx.common.vertx.ext.web.handler.impl.OutputTrxInfoOnErrorHandlerImpl.handle(OutputTrxInfoOnErrorHandlerImpl.java:18) at com.xxxxxx.common.vertx.ext.web.handler.impl.OutputTrxInfoOnErrorHandlerImpl.handle(OutputTrxInfoOnErrorHandlerImpl.java:10) at io.vertx.ext.web.impl.RouteState.handleContext(RouteState.java:1034) at io.vertx.ext.web.impl.RoutingContextImplBase.iterateNext(RoutingContextImplBase.java:95) at io.vertx.ext.web.impl.RoutingContextImpl.next(RoutingContextImpl.java:128) at com.xxxxxx.common.vertx.ext.web.handler.impl.TransactionDateTimeHandlerImpl.handle(TransactionDateTimeHandlerImpl.java:19) at com.xxxxxx.common.vertx.ext.web.handler.impl.TransactionDateTimeHandlerImpl.handle(TransactionDateTimeHandlerImpl.java:11) at io.vertx.ext.web.impl.RouteState.handleContext(RouteState.java:1034) at io.vertx.ext.web.impl.RoutingContextImplBase.iterateNext(RoutingContextImplBase.java:131) at io.vertx.ext.web.impl.RoutingContextImpl.next(RoutingContextImpl.java:128) at io.vertx.ext.web.impl.RouterImpl.handle(RouterImpl.java:54) at io.vertx.ext.web.impl.RouterImpl.handle(RouterImpl.java:36) at io.vertx.ext.web.Router.accept(Router.java:64) at io.vertx.core.http.impl.WebSocketRequestHandler.handle(WebSocketRequestHandler.java:50) at io.vertx.core.http.impl.WebSocketRequestHandler.handle(WebSocketRequestHandler.java:32) at io.vertx.core.http.impl.Http1xServerConnection.handleMessage(Http1xServerConnection.java:136) at io.vertx.core.impl.ContextImpl.executeTask(ContextImpl.java:369) at io.vertx.core.impl.EventLoopContext.execute(EventLoopContext.java:43) at io.vertx.core.impl.ContextImpl.executeFromIO(ContextImpl.java:232) at io.vertx.core.net.impl.VertxHandler.channelRead(VertxHandler.java:173) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:93) at io.netty.handler.codec.http.websocketx.extensions.WebSocketServerExtensionHandler.channelRead(WebSocketServerExtensionHandler.java:101) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:321) at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:295) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:714) at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650) at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ... 4 common frames omitted
