Hi Vladimir,
My table is very simple, it contains the following information
OrgId (varchar) , oId(varchar), fnum(int), gId(number), msg(varchar),
num(number), date(date)
the gId is the primary Index.
--------------------------------------------------------------------------------------------------
And the java class is defined as:
package com.huawei.soa.ignite.test;
import java.io.Serializable;
import java.math.BigDecimal;
import java.util.Date;
import org.apache.ignite.cache.query.annotations.QuerySqlField;
public class UniqueField implements Serializable
{
public String getOrgId()
{
return orgId;
}
public void setOrgId(String orgId)
{
this.orgId = orgId;
}
public String getOId()
{
return oId;
}
public void setOId(String oId)
{
this.oId = oId;
}
public String getGid()
{
return gId;
}
public void setGuid(String gId)
{
this.gId = gId;
}
public int getFNum()
{
return fNum;
}
public void setFNum(int fNum)
{
this.fNum = fNum;
}
public String getMsg()
{
return msg;
}
public void setMsg(String msg)
{
this.msg = msg;
}
public BigDecimal getNum()
{
return num;
}
public void setNum(BigDecimal num)
{
this.num = num;
}
public Date getDate()
{
return date;
}
public void setDate(Date date)
{
this.date = date;
}
@QuerySqlField
private String orgId;
@QuerySqlField(index=true)
private String oId;
@QuerySqlField(index=true)
private String gId;
@QuerySqlField
private int fNum;
@QuerySqlField
private String msg;
@QuerySqlField
private BigDecimal num;
@QuerySqlField
private Date date;
public UniqueField(){};
public UniqueField(
String orgId,
String oId,
String gId,
int fNum,
String msg,
BigDecimal num,
Date date
){
this.orgId=orgId;
this.oId=oId;
this.gId = gId;
this.fNum = fNum;
this.msg = msg;
this.num = num;
this.date = date;
}
}
--------------------------------------------------------------------------------------------------
My configuration file on the server side is listed as follows:
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<beans xmlns="http://www.springframework.org/schema/beans";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="
http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans.xsd";>
<bean id="dataSource" class="oracle.jdbc.pool.OracleDataSource">
<!-- real data is not listed -->
<property name="URL" value="jdbc:oracle:thin:@xxx.xxx.xxx.xxx:xxx" />
<property name="user" value="xxx"/>
<property name="password" value="xxx" />
</bean>
<!--
Alter configuration below as needed.
-->
<bean id="grid1.cfg"
class="org.apache.ignite.configuration.IgniteConfiguration">
<property name="peerClassLoadingEnabled" value="true" />
<property name="cacheConfiguration">
<list>
<bean
class="org.apache.ignite.configuration.CacheConfiguration">
<property name="cacheStoreFactory">
<!-- CacheJdbcPojoStoreExampleFactory is
extend from CacheJdbcPojoStoreFactory -->
<bean
class="com.huawei.soa.ignite.util.CacheJdbcPojoStoreExampleFactory">
<property name="dataSourceBean"
value="dataSource" />
<property name="dialect" >
<bean
class="org.apache.ignite.cache.store.jdbc.dialect.OracleDialect"/>
</property>
</bean>
</property>
<!-- defintion from ignite document -->
<property name="memoryMode" value="OFFHEAP_TIERED"
/>
<!-- 10g offheap allocated-->
<property name="offHeapMaxMemory"
value="#{10*1024L*1024L*1024L}" />
<property name="readThrough" value="true" />
<property name="writeThrough" value="true" />
<property name="startSize" value="#{200*1024*1024}"
/>
<property name="cacheMode" value="PARTITIONED" />
<property name="backups" value="0" />
</bean>
</list>
</property>
<property name="discoverySpi">
<bean
class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
<property name="ipFinder" >
<bean
class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder">
<property name="addresses">
<list>
<value>
xxx.xxx.xxx.xxx:47500..47509</value>
<value>
xxx.xxx.xxx.xxx:47500..47509</value>
<value>
xxx.xxx.xxx.xxx:47500..47509 </value>
<value>
xxx.xxx.xxx.xxx:47500..47509 </value>
</list>
</property>
</bean>
</property>
</bean>
</property>
</bean>
</beans>
Kind regards,
Kevin
发件人: Vladimir Ozerov [mailto:[email protected]]
发送时间: 2016年4月22日 20:13
收件人: [email protected]
主题: Re: Ignite cache data size problem.
Hi,
It looks like you have relatively small entries (somewhere about 60-70 bytes
per key-value pair). Ignite also has some intrinsic overhead, which could be
more than actual data in this case. However, I surely would not expect that it
will not fit into 80GB.
Could you please share your key and value model classes and your XML
configuration to investigate it further?
Vladimir.
On Fri, Apr 22, 2016 at 2:02 PM, Zhengqingzheng
<[email protected]<mailto:[email protected]>> wrote:
Hi there,
I am trying to load a table with 47Million records, in which the data size is
less than 3gb. However, When I load into the memory ( two vm with 48+32 =
80gb), it is still crushed due to not enough memory space? This problem
occurred when I instantiated 6 + 4 nodes.
Why the cache model need so much space ( 3g vs 80g)?
Any idea to explain this issue?
Kind regards,
Kevin
