hi,
Bought 4 ignite servers on k8s, and ingested 160GB of data.
I have a java application which is used to connect to servers as a client
to get the metrics from the servers.
Client will be up , but after some time it will get disconnect.
In the server logs i get below error:
{"type":"log","host":"ignite-cluster-0","level":"INFO","systemid":"5605ca0e","system":"ignite-service","time":"2019-02-06
09:24:34,201","logger":"GridDiscoveryManager","timezone":"UTC","marker":"","log":"Added
new node to topology: TcpDiscoveryNode
[id=6daf3474-2265-48dd-a8f0-471d1a437084, addrs=[0:0:0:0:0:0:0:1%lo,
127.0.0.1, 192.168.1.165], sockAddrs=[/0:0:0:0:0:0:0:1%lo:0, /127.0.0.1:0, /
192.168.1.165:0], discPort=0, order=2148, intOrder=1077,
lastExchangeTime=1549445074167, loc=false,
ver=2.6.0#20180710-sha1:669feacc, isClient=true]"}
{"type":"log","host":"ignite-cluster-0","level":"INFO","systemid":"5605ca0e","system":"ignite-service","time":"2019-02-06
09:24:34,201","logger":"time","timezone":"UTC","marker":"","log":"Started
exchange init [topVer=AffinityTopologyVersion [topVer=2148, minorTopVer=0],
crd=true, evt=NODE_JOINED, evtNode=6daf3474-2265-48dd-a8f0-471d1a437084,
customEvt=null, allowMerge=true]"}
{"type":"log","host":"ignite-cluster-0","level":"INFO","systemid":"5605ca0e","system":"ignite-service","time":"2019-02-06
09:24:34,214","logger":"GridCachePartitionExchangeManager","timezone":"UTC","marker":"","log":"Skipping
rebalancing (nothing scheduled) [top=AffinityTopologyVersion [topVer=2148,
minorTopVer=0], evt=NODE_JOINED,
node=6daf3474-2265-48dd-a8f0-471d1a437084]"}
{"type":"log","host":"ignite-cluster-0","level":"INFO","systemid":"5605ca0e","system":"ignite-service","time":"2019-02-06
09:24:41,981","logger":"TcpDiscoverySpi","timezone":"UTC","marker":"","log":"Pinging
node: 6daf3474-2265-48dd-a8f0-471d1a437084"}
{"type":"log","host":"ignite-cluster-0","level":"INFO","systemid":"5605ca0e","system":"ignite-service","time":"2019-02-06
09:24:41,985","logger":"TcpDiscoverySpi","timezone":"UTC","marker":"","log":"Finished
node ping [nodeId=6daf3474-2265-48dd-a8f0-471d1a437084, res=false,
time=5ms]"}
{"type":"log","host":"ignite-cluster-0","level":"WARN","systemid":"5605ca0e","system":"ignite-service","time":"2019-02-06
09:25:11,331","logger":"TcpDiscoverySpi","timezone":"UTC","marker":"","log":"Failing
client node due to not receiving metrics updates from client node within
'IgniteConfiguration.clientFailureDetectionTimeout' (consider increasing
configuration property) [timeout=30000, node=TcpDiscoveryNode
[id=6daf3474-2265-48dd-a8f0-471d1a437084, addrs=[0:0:0:0:0:0:0:1%lo,
127.0.0.1, 192.168.1.165], sockAddrs=[/0:0:0:0:0:0:0:1%lo:0, /127.0.0.1:0, /
192.168.1.165:0], discPort=0, order=2148, intOrder=1077,
lastExchangeTime=1549445074167, loc=false,
ver=2.6.0#20180710-sha1:669feacc, isClient=true]]"}
{"type":"log","host":"ignite-cluster-0","level":"WARN","systemid":"5605ca0e","system":"ignite-service","time":"2019-02-06
09:25:11,332","logger":"GridDiscoveryManager","timezone":"UTC","marker":"","log":"Node
FAILED: TcpDiscoveryNode [id=6daf3474-2265-48dd-a8f0-471d1a437084,
addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 192.168.1.165],
sockAddrs=[/0:0:0:0:0:0:0:1%lo:0, /127.0.0.1:0, /192.168.1.165:0],
discPort=0, order=2148, intOrder=1077, lastExchangeTime=1549445074167,
loc=false, ver=2.6.0#20180710-sha1:669feacc, isClient=true]"}
{"type":"log","host":"ignite-cluster-0","level":"INFO","systemid":"5605ca0e","system":"ignite-service","time":"2019-02-06
09:25:11,333","logger":"time","timezone":"UTC","marker":"","log":"Started
exchange init [topVer=AffinityTopologyVersion [topVer=2151, minorTopVer=0],
crd=true, evt=NODE_FAILED, evtNode=6daf3474-2265-48dd-a8f0-471d1a437084,
customEvt=null, allowMerge=true]"}
{"type":"log","host":"ignite-cluster-0","level":"INFO","systemid":"5605ca0e","system":"ignite-service","time":"2019-02-06
09:25:11,349","logger":"GridCachePartitionExchangeManager","timezone":"UTC","marker":"","log":"Skipping
rebalancing (nothing scheduled) [top=AffinityTopologyVersion [topVer=2151,
minorTopVer=0], evt=NODE_FAILED,
node=6daf3474-2265-48dd-a8f0-471d1a437084]"}
Client config file:
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xmlns:util="http://www.springframework.org/schema/util";
xsi:schemaLocation="
http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/util
http://www.springframework.org/schema/util/spring-util.xsd";>
<bean class="org.apache.ignite.configuration.IgniteConfiguration">
<property name="connectorConfiguration">
<bean
class="org.apache.ignite.configuration.ConnectorConfiguration">
<property name="jettyPath"
value="/opt/ignite/conf/jetty-server.xml" />
</bean>
</property>
<property name="peerClassLoadingEnabled" value="true"/>
<!-- Enable cache events. -->
<property name="includeEventTypes">
<util:constant
static-field="org.apache.ignite.events.EventType.EVTS_DISCOVERY"/>
</property>
<property name="failureDetectionTimeout" value="80000"/>
<property name="clientFailureDetectionTimeout" value="80000"/>
<property name="discoverySpi">
<bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
<property name="ipFinder">
<bean
class="org.apache.ignite.spi.discovery.tcp.ipfinder.kubernetes.TcpDiscoveryKubernetesIpFinder">
<property name="serviceName" value="ignite-service"/>
<property name="namespace" value="ign"/>
</bean>
</property>
<property name="networkTimeout" value="60000"/>
</bean>
</property>
</bean>
</beans>
Thanks
Radha
