Hi team,
I'm getting strange exception when trying to save Key/Value pair.
When I tested locally everything was fine.
Now I deployed application to servers cluster and stuck with this error.
What does it mean? My both Ignite server nodes are UP.
Here is configuration for this cache:
<bean class="org.apache.ignite.configuration.CacheConfiguration">
<property name="name" value="RefreshToken"/>
<property name="atomicityMode" value="TRANSACTIONAL"/>
<property name="writeSynchronizationMode"
value="FULL_SYNC"/>
<property name="cacheMode" value="PARTITIONED"/>
<property name="backups" value="0"/>
</bean>
Here is my topology
visor> top
Hosts: 4
+===============================================================================================================================+
| Int./Ext. IPs | Node ID8(@) | Node Type |
OS | CPUs | MACs | CPU Load |
+===============================================================================================================================+
| 0:0:0:0:0:0:0:1 | 1: 35CD3AD1(@n2) | Client | Windows Server 2012 R2
amd64 6.3 | 4 | 00:00:00:00:00:00:00:E0 | 0.00 % |
| 10.2.0.225 | |
| | | 00:50:56:25:00:B9
| |
| 127.0.0.1 | |
| | |
| |
| 30.251.106.197 | |
| | |
| |
+-----------------+------------------+-----------+----------------------------------+------+-------------------------+----------+
| 0:0:0:0:0:0:0:1 | 1: 230F83B8(@n3) | Client | Windows Server 2012 R2
amd64 6.3 | 4 | 00:00:00:00:00:00:00:E0 | 0.13 % |
| 10.2.0.250 | |
| | | 00:50:56:25:00:35
| |
| 127.0.0.1 | |
| | |
| |
| 30.251.106.11 | |
| | |
| |
+-----------------+------------------+-----------+----------------------------------+------+-------------------------+----------+
| 0:0:0:0:0:0:0:1 | 1: DF9FD2A4(@n0) | Server | Windows Server 2012 R2
amd64 6.3 | 4 | 00:00:00:00:00:00:00:E0 | 0.00 % |
| 10.2.0.163 | |
| | | 00:50:56:25:00:B7
| |
| 127.0.0.1 | |
| | |
| |
| 30.251.106.199 | |
| | |
| |
+-----------------+------------------+-----------+----------------------------------+------+-------------------------+----------+
| 0:0:0:0:0:0:0:1 | 1: 965AD7CD(@n1) | Server | Windows Server 2012 R2
amd64 6.3 | 4 | 00:00:00:00:00:00:00:E0 | 0.00 % |
| 10.2.0.252 | |
| | | 00:50:56:25:00:BB
| |
| 127.0.0.1 | |
| | |
| |
| 30.251.106.90 | |
| | |
| |
+-------------------------------------------------------------------------------------------------------------------------------+
REQ_002 2018-04-05 23:08:05.259 [async-worker-2] ERROR
com.xxx.backend.async.AsyncExecutor - Failed to lock keys (all partition
nodes left the grid).
org.apache.ignite.cache.CacheServerNotFoundException: Failed to lock keys
(all partition nodes left the grid).
at
org.apache.ignite.internal.processors.cache.GridCacheUtils.convertToCacheException(GridCacheUtils.java:1282)
~[ignite-core-2.4.0.jar:2.4.0]
at
org.apache.ignite.internal.processors.cache.IgniteCacheProxyImpl.cacheException(IgniteCacheProxyImpl.java:1673)
~[ignite-core-2.4.0.jar:2.4.0]
at
org.apache.ignite.internal.processors.cache.IgniteCacheProxyImpl.get(IgniteCacheProxyImpl.java:852)
~[ignite-core-2.4.0.jar:2.4.0]
at
org.apache.ignite.internal.processors.cache.GatewayProtectedCacheProxy.get(GatewayProtectedCacheProxy.java:676)
~[ignite-core-2.4.0.jar:2.4.0]
at
com.xxx.backend.cache.impl.DurableCacheIgnite.storeRefreshToken(DurableCacheIgnite.java:268)
~[lk.backend-1.0.0.jar:?]
at
com.xxx.cache.impl.DurableCacheIgnite$$FastClassBySpringCGLIB$$fa2c0515.invoke(<generated>)
~[lk.backend-1.0.0.jar:?]
at
org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204)
~[spring-core-5.0.4.RELEASE.jar:5.0.4.RELEASE]
at
org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:747)
~[spring-aop-5.0.4.RELEASE.jar:5.0.4.RELEASE]
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163)
~[spring-aop-5.0.4.RELEASE.jar:5.0.4.RELEASE]
at
org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodInvocationProceedingJoinPoint.java:89)
~[spring-aop-5.0.4.RELEASE.jar:5.0.4.RELEASE]
at
com.xxx.backend.logging.ComponentLogger.logTimeMethod(ComponentLogger.java:34)
~[lk.backend-1.0.0.jar:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
~[?:1.8.0_162]
at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
~[?:1.8.0_162]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
~[?:1.8.0_162]
at java.lang.reflect.Method.invoke(Unknown Source) ~[?:1.8.0_162]
at
org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:643)
~[spring-aop-5.0.4.RELEASE.jar:5.0.4.RELEASE]
at
org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:632)
~[spring-aop-5.0.4.RELEASE.jar:5.0.4.RELEASE]
at
org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java:70)
~[spring-aop-5.0.4.RELEASE.jar:5.0.4.RELEASE]
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:185)
~[spring-aop-5.0.4.RELEASE.jar:5.0.4.RELEASE]
at
org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:92)
~[spring-aop-5.0.4.RELEASE.jar:5.0.4.RELEASE]
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:185)
~[spring-aop-5.0.4.RELEASE.jar:5.0.4.RELEASE]
at
org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:689)
~[spring-aop-5.0.4.RELEASE.jar:5.0.4.RELEASE]
at
com.xxx.backend.cache.impl.DurableCacheIgnite$$EnhancerBySpringCGLIB$$29ce246.storeRefreshToken(<generated>)
~[lk.backend-1.0.0.jar:?]
at
com.xxx.backend.ctl.person.PersonRegistrationController.lambda$registerUser$1(PersonRegistrationController.java:88)
~[lk.backend-1.0.0.jar:?]
at
com.xxx.backend.async.AsyncExecutor$AsyncTask.run(AsyncExecutor.java:98)
[lk.backend-1.0.0.jar:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
[?:1.8.0_162]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
[?:1.8.0_162]
at java.lang.Thread.run(Unknown Source) [?:1.8.0_162]
Caused by:
org.apache.ignite.internal.cluster.ClusterTopologyServerNotFoundException:
Failed to lock keys (all partition nodes left the grid).
at
org.apache.ignite.internal.processors.cache.distributed.dht.colocated.GridDhtColocatedLockFuture.map(GridDhtColocatedLockFuture.java:1383)
~[ignite-core-2.4.0.jar:2.4.0]
at
org.apache.ignite.internal.processors.cache.distributed.dht.colocated.GridDhtColocatedLockFuture.map0(GridDhtColocatedLockFuture.java:904)
~[ignite-core-2.4.0.jar:2.4.0]
at
org.apache.ignite.internal.processors.cache.distributed.dht.colocated.GridDhtColocatedLockFuture.map(GridDhtColocatedLockFuture.java:856)
~[ignite-core-2.4.0.jar:2.4.0]
at
org.apache.ignite.internal.processors.cache.distributed.dht.colocated.GridDhtColocatedLockFuture.mapOnTopology(GridDhtColocatedLockFuture.java:815)
~[ignite-core-2.4.0.jar:2.4.0]
at
org.apache.ignite.internal.processors.cache.distributed.dht.colocated.GridDhtColocatedLockFuture.map(GridDhtColocatedLockFuture.java:763)
~[ignite-core-2.4.0.jar:2.4.0]
at
org.apache.ignite.internal.processors.cache.distributed.dht.colocated.GridDhtColocatedCache.lockAllAsync(GridDhtColocatedCache.java:646)
~[ignite-core-2.4.0.jar:2.4.0]
at
org.apache.ignite.internal.processors.cache.distributed.GridDistributedCacheAdapter.txLockAsync(GridDistributedCacheAdapter.java:109)
~[ignite-core-2.4.0.jar:2.4.0]
at
org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.getAllAsync(GridNearTxLocal.java:1723)
~[ignite-core-2.4.0.jar:2.4.0]
at
org.apache.ignite.internal.processors.cache.distributed.dht.colocated.GridDhtColocatedCache$4.op(GridDhtColocatedCache.java:197)
~[ignite-core-2.4.0.jar:2.4.0]
at
org.apache.ignite.internal.processors.cache.GridCacheAdapter$AsyncOp.op(GridCacheAdapter.java:5140)
~[ignite-core-2.4.0.jar:2.4.0]
at
org.apache.ignite.internal.processors.cache.GridCacheAdapter.asyncOp(GridCacheAdapter.java:4275)
~[ignite-core-2.4.0.jar:2.4.0]
at
org.apache.ignite.internal.processors.cache.distributed.dht.colocated.GridDhtColocatedCache.getAsync(GridDhtColocatedCache.java:195)
~[ignite-core-2.4.0.jar:2.4.0]
at
org.apache.ignite.internal.processors.cache.GridCacheAdapter.get0(GridCacheAdapter.java:4565)
~[ignite-core-2.4.0.jar:2.4.0]
at
org.apache.ignite.internal.processors.cache.GridCacheAdapter.get(GridCacheAdapter.java:4546)
~[ignite-core-2.4.0.jar:2.4.0]
at
org.apache.ignite.internal.processors.cache.GridCacheAdapter.get(GridCacheAdapter.java:1347)
~[ignite-core-2.4.0.jar:2.4.0]
at
org.apache.ignite.internal.processors.cache.IgniteCacheProxyImpl.get(IgniteCacheProxyImpl.java:849)
~[ignite-core-2.4.0.jar:2.4.0]
... 25 more
