Re: Missing records Ignite cache size grows

[Andrey Mashenkov]

Hi diopek,

I can't see how cache size relates to value classes and value object
content.

I rework my test and still can't reproduce this issue even with 2 millions
of entries in cache, but you mention about a half a million.

On Fri, Mar 3, 2017 at 6:25 PM, diopek <deha.pe...@gmail.com> wrote:

> Hi Andrew,
> Difference in your repro;
> your cache entry is <Long, ArrayList&lt;String>>,
> and you always add single entry to value ArrayList
> in our case
> <Long, ArrayList&lt;MyPOJO>>
> MyPOJO is java bean that has ~30 attributes.
> and ArrayList might have sometimes 100s of objects.
> During my local tests, that I am using ~20GB of RAM, I couldn't reproduce
> either.
> Issue occurs with high number of records with production data on servers
> with larger amount of RAM.
> I am wondering if you can try to replicate that scenario on your end.
> Thanks
>
>
>
>
>
>
> --
> View this message in context: http://apache-ignite-users.
> 70518.x6.nabble.com/Missing-records-Ignite-cache-size-
> grows-tp10809p11022.html
> Sent from the Apache Ignite Users mailing list archive at Nabble.com.
>



-- 
Best regards,
Andrey V. Mashenkov

import java.io.Serializable;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.concurrent.ThreadLocalRandom;
import java.util.stream.LongStream;
import javax.cache.configuration.FactoryBuilder;
import javax.cache.integration.CacheLoaderException;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
import org.apache.ignite.cache.CacheMode;
import org.apache.ignite.cache.store.CacheLoadOnlyStoreAdapter;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.lang.IgniteBiTuple;

/**
 * Can't reproduced.
 */
public class MissedEntries {
    /** */
    private static final int CACHE_SIZE = 2_000_000;

    /** */
    private static CacheConfiguration cacheConfig(String cacheName) {
        CacheConfiguration<Long, ArrayList> cacheCfg = new CacheConfiguration<>(cacheName);
        cacheCfg.setStartSize(CACHE_SIZE);
        cacheCfg.setCacheMode(CacheMode.LOCAL);
        cacheCfg.setIndexedTypes(Long.class, ArrayList.class);

        cacheCfg.setCacheStoreFactory(FactoryBuilder.factoryOf(MyCacheLoadOnlyStore.class));

        return cacheCfg;
    }

    /** */
    public static void main(String[] args) {
        try {
            Ignite ignite = Ignition.start(new IgniteConfiguration().setGridName("node-0"));

            IgniteCache cache = ignite.getOrCreateCache(cacheConfig("myCache"));

            cache.loadCache(null);

            int size = cache.size();

            if (CACHE_SIZE != size)
                throw new AssertionError("Data lost: "+size);
        }
        finally {
            Ignition.stopAll(true);
        }
    }

    /**
     *
     */
    public static class MyCacheLoadOnlyStore extends CacheLoadOnlyStoreAdapter<Long, ArrayList, Long> {
        /** {@inheritDoc} */
        @Override protected Iterator<Long> inputIterator(Object... args) throws CacheLoaderException {
            return LongStream.range(0, CACHE_SIZE).iterator();
        }

        /** {@inheritDoc} */
        @Override protected IgniteBiTuple<Long, ArrayList> parse(Long rec, Object... args) {
            final int size = ThreadLocalRandom.current().nextInt(50);

            ArrayList<MyObj> val1 = new ArrayList<>(size);

            for (int i = 0; i < size; i++)
                val1.add(MyObj.createRandom());

            if(rec % 10_000 == 0)
                System.out.println("Generated: "+rec);

            return new IgniteBiTuple<>(rec, val1);
        }
    }

    public static class MyObj implements Serializable {
        int field1;

        int field2;

        int field3;

        String field4;

        String field5;

        public MyObj(int field1, int field2, int field3, String field4, String field5) {
            this.field1 = field1;
            this.field2 = field2;
            this.field3 = field3;
            this.field4 = field4;
            this.field5 = field5;
        }

        static MyObj createRandom() {
            final ThreadLocalRandom random = ThreadLocalRandom.current();

            return new MyObj(
                random.nextInt(20),
                random.nextInt(50, 70),
                random.nextInt(100, 120),
                String.valueOf(random.nextInt(100)),
                String.valueOf(random.nextInt(1000))
            );
        }
    }
}