http://git-wip-us.apache.org/repos/asf/ignite/blob/81ae2d83/leak.patch ---------------------------------------------------------------------- diff --cc leak.patch index 0000000,0000000..f618073 new file mode 100644 --- /dev/null +++ b/leak.patch @@@ -1,0 -1,0 +1,10521 @@@ ++From 3c9e3786a21ec7c8919de9c29bc04d2e3561846c Mon Sep 17 00:00:00 2001 ++From: Igor Seliverstov <[email protected]> ++Date: Wed, 15 Feb 2017 13:41:08 +0300 ++Subject: [PATCH 01/41] IGNITE-4694 Add tests to check there are no memory ++ leaks in PageMemory ++ ++--- ++ .../processors/database/IgniteDbAbstractTest.java | 360 +++++++++++++++++++++ ++ .../database/IgniteDbMemoryLeakAbstractTest.java | 84 +++++ ++ .../database/IgniteDbMemoryLeakIndexedTest.java | 85 +++++ ++ .../IgniteDbMemoryLeakLargeObjectsTest.java | 95 ++++++ ++ .../database/IgniteDbMemoryLeakLargePagesTest.java | 90 ++++++ ++ .../database/IgniteDbMemoryLeakTest.java | 85 +++++ ++ .../IgniteDbMemoryLeakWithExpirationTest.java | 92 ++++++ ++ .../database/IgniteDbPutGetAbstractTest.java | 347 +------------------- ++ 8 files changed, 903 insertions(+), 335 deletions(-) ++ create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java ++ create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java ++ create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java ++ create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java ++ create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java ++ create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java ++ create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java ++ ++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java ++new file mode 100644 ++index 0000000..3bc7004 ++--- /dev/null +++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java ++@@ -0,0 +1,360 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++ +++package org.apache.ignite.internal.processors.database; +++ +++import org.apache.ignite.cache.CacheAtomicityMode; +++import org.apache.ignite.cache.CacheRebalanceMode; +++import org.apache.ignite.cache.CacheWriteSynchronizationMode; +++import org.apache.ignite.cache.affinity.AffinityFunction; +++import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +++import org.apache.ignite.cache.query.annotations.QuerySqlField; +++import org.apache.ignite.configuration.CacheConfiguration; +++import org.apache.ignite.configuration.IgniteConfiguration; +++import org.apache.ignite.configuration.MemoryConfiguration; +++import org.apache.ignite.internal.processors.cache.database.tree.BPlusTree; +++import org.apache.ignite.internal.util.typedef.internal.S; +++import org.apache.ignite.internal.util.typedef.internal.U; +++import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +++import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; +++import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +++import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +++ +++import java.io.Serializable; +++import java.util.Arrays; +++import java.util.Random; +++ +++/** +++ * +++ */ +++public abstract class IgniteDbAbstractTest extends GridCommonAbstractTest { +++ /** */ +++ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); +++ +++ /** +++ * @return Node count. +++ */ +++ protected abstract int gridCount(); +++ +++ /** +++ * @return {@code True} if indexing is enabled. +++ */ +++ protected abstract boolean indexingEnabled(); +++ +++ /** {@inheritDoc} */ +++ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { +++ IgniteConfiguration cfg = super.getConfiguration(gridName); +++ +++ MemoryConfiguration dbCfg = new MemoryConfiguration(); +++ +++ dbCfg.setConcurrencyLevel(Runtime.getRuntime().availableProcessors() * 4); +++ +++ if (isLargePage()) +++ dbCfg.setPageSize(16 * 1024); +++ else +++ dbCfg.setPageSize(1024); +++ +++ +++ dbCfg.setPageCacheSize(200 * 1024 * 1024); +++ +++ configure(dbCfg); +++ +++ cfg.setMemoryConfiguration(dbCfg); +++ +++ CacheConfiguration ccfg = new CacheConfiguration(); +++ +++ if (indexingEnabled()) +++ ccfg.setIndexedTypes(Integer.class, DbValue.class); +++ +++ ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); +++ ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); +++ ccfg.setRebalanceMode(CacheRebalanceMode.SYNC); +++ ccfg.setAffinity(new RendezvousAffinityFunction(false, 32)); +++ +++ CacheConfiguration ccfg2 = new CacheConfiguration("non-primitive"); +++ +++ if (indexingEnabled()) +++ ccfg2.setIndexedTypes(DbKey.class, DbValue.class); +++ +++ ccfg2.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); +++ ccfg2.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); +++ ccfg2.setRebalanceMode(CacheRebalanceMode.SYNC); +++ ccfg2.setAffinity(new RendezvousAffinityFunction(false, 32)); +++ +++ CacheConfiguration ccfg3 = new CacheConfiguration("large"); +++ +++ if (indexingEnabled()) +++ ccfg3.setIndexedTypes(Integer.class, LargeDbValue.class); +++ +++ ccfg3.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); +++ ccfg3.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); +++ ccfg3.setRebalanceMode(CacheRebalanceMode.SYNC); +++ ccfg3.setAffinity(new RendezvousAffinityFunction(false, 32)); +++ +++ CacheConfiguration ccfg4 = new CacheConfiguration("tiny"); +++ +++ ccfg4.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); +++ ccfg4.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); +++ ccfg4.setRebalanceMode(CacheRebalanceMode.SYNC); +++ ccfg4.setAffinity(new RendezvousAffinityFunction(false, 32)); +++ +++ final AffinityFunction aff = new RendezvousAffinityFunction(1, null); +++ +++ ccfg4.setAffinity(aff); +++ +++ cfg.setCacheConfiguration(ccfg, ccfg2, ccfg3, ccfg4); +++ +++ TcpDiscoverySpi discoSpi = new TcpDiscoverySpi(); +++ +++ discoSpi.setIpFinder(IP_FINDER); +++ +++ cfg.setDiscoverySpi(discoSpi); +++ cfg.setMarshaller(null); +++ +++ configure(cfg); +++ +++ return cfg; +++ } +++ +++ protected void configure(IgniteConfiguration cfg){ +++ //NOP +++ } +++ +++ protected void configure(MemoryConfiguration mCfg){ +++ //NOP +++ } +++ +++ /** {@inheritDoc} */ +++ @Override protected void beforeTest() throws Exception { +++ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false)); +++ +++ long seed = 1464583813940L; // System.currentTimeMillis(); +++ +++ info("Seed: " + seed + "L"); +++ +++ BPlusTree.rnd = new Random(seed); +++ +++ startGrids(gridCount()); +++ +++ awaitPartitionMapExchange(); +++ } +++ +++ /** {@inheritDoc} */ +++ @Override protected void afterTest() throws Exception { +++ BPlusTree.rnd = null; +++ +++ stopAllGrids(); +++ +++ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false)); +++ } +++ +++ /** +++ * @return {@code True} if use large page. +++ */ +++ protected boolean isLargePage() { +++ return false; +++ } +++ +++ /** +++ * +++ */ +++ static class DbKey implements Serializable { +++ /** */ +++ int val; +++ +++ /** +++ * @param val Value. +++ */ +++ DbKey(int val) { +++ this.val = val; +++ } +++ +++ /** {@inheritDoc} */ +++ @Override public boolean equals(Object o) { +++ if (this == o) +++ return true; +++ +++ if (o == null || !(o instanceof DbKey)) +++ return false; +++ +++ DbKey key = (DbKey)o; +++ +++ return val == key.val; +++ } +++ +++ /** {@inheritDoc} */ +++ @Override public int hashCode() { +++ return val; +++ } +++ } +++ +++ /** +++ * +++ */ +++ static class LargeDbKey implements Serializable { +++ /** */ +++ int val; +++ +++ /** */ +++ byte[] data; +++ +++ /** +++ * @param val Value. +++ * @param size Key payload size. +++ */ +++ LargeDbKey(int val, int size) { +++ this.val = val; +++ +++ data = new byte[size]; +++ +++ Arrays.fill(data, (byte)val); +++ } +++ +++ /** {@inheritDoc} */ +++ @Override public boolean equals(Object o) { +++ if (this == o) +++ return true; +++ +++ if (o == null || !(o instanceof LargeDbKey)) +++ return false; +++ +++ LargeDbKey key = (LargeDbKey)o; +++ +++ return val == key.val && Arrays.equals(data, key.data); +++ } +++ +++ /** {@inheritDoc} */ +++ @Override public int hashCode() { +++ return val + Arrays.hashCode(data); +++ } +++ } +++ +++ /** +++ * +++ */ +++ static class DbValue implements Serializable { +++ /** */ +++ @QuerySqlField(index = true) +++ int iVal; +++ +++ /** */ +++ @QuerySqlField(index = true) +++ String sVal; +++ +++ /** */ +++ @QuerySqlField +++ long lVal; +++ +++ +++ +++ /** +++ * @param iVal Integer value. +++ * @param sVal String value. +++ * @param lVal Long value. +++ */ +++ DbValue(int iVal, String sVal, long lVal) { +++ this.iVal = iVal; +++ this.sVal = sVal; +++ this.lVal = lVal; +++ } +++ +++ /** {@inheritDoc} */ +++ @Override public boolean equals(Object o) { +++ if (this == o) +++ return true; +++ +++ if (o == null || getClass() != o.getClass()) +++ return false; +++ +++ DbValue dbVal = (DbValue)o; +++ +++ return iVal == dbVal.iVal && lVal == dbVal.lVal && +++ !(sVal != null ? !sVal.equals(dbVal.sVal) : dbVal.sVal != null); +++ } +++ +++ /** {@inheritDoc} */ +++ @Override public int hashCode() { +++ int res = iVal; +++ +++ res = 31 * res + (sVal != null ? sVal.hashCode() : 0); +++ res = 31 * res + (int)(lVal ^ (lVal >>> 32)); +++ +++ return res; +++ } +++ +++ /** {@inheritDoc} */ +++ @Override public String toString() { +++ return S.toString(DbValue.class, this); +++ } +++ } +++ +++ /** +++ * +++ */ +++ static class LargeDbValue { +++ /** */ +++ @QuerySqlField(index = true) +++ String str1; +++ +++ /** */ +++ @QuerySqlField(index = true) +++ String str2; +++ +++ /** */ +++ int[] arr; +++ +++ /** +++ * @param str1 String 1. +++ * @param str2 String 2. +++ * @param arr Big array. +++ */ +++ LargeDbValue(final String str1, final String str2, final int[] arr) { +++ this.str1 = str1; +++ this.str2 = str2; +++ this.arr = arr; +++ } +++ +++ /** {@inheritDoc} */ +++ @Override public boolean equals(final Object o) { +++ if (this == o) return true; +++ if (o == null || getClass() != o.getClass()) return false; +++ +++ final LargeDbValue that = (LargeDbValue) o; +++ +++ if (str1 != null ? !str1.equals(that.str1) : that.str1 != null) return false; +++ if (str2 != null ? !str2.equals(that.str2) : that.str2 != null) return false; +++ +++ return Arrays.equals(arr, that.arr); +++ +++ } +++ +++ /** {@inheritDoc} */ +++ @Override public int hashCode() { +++ int res = str1 != null ? str1.hashCode() : 0; +++ +++ res = 31 * res + (str2 != null ? str2.hashCode() : 0); +++ res = 31 * res + Arrays.hashCode(arr); +++ +++ return res; +++ } +++ +++ /** {@inheritDoc} */ +++ @Override public String toString() { +++ return S.toString(LargeDbValue.class, this); +++ } +++ } +++} ++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java ++new file mode 100644 ++index 0000000..6a5d039 ++--- /dev/null +++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java ++@@ -0,0 +1,84 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++ +++package org.apache.ignite.internal.processors.database; +++ +++import org.apache.ignite.Ignite; +++import org.apache.ignite.IgniteCompute; +++import org.apache.ignite.compute.ComputeTaskFuture; +++import org.apache.ignite.internal.IgniteEx; +++import org.apache.ignite.lang.IgniteRunnable; +++import org.apache.ignite.resources.IgniteInstanceResource; +++ +++import java.util.concurrent.TimeUnit; +++ +++/** +++ * +++ */ +++public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTest { +++ +++ /** Test duration in seconds*/ +++ protected abstract int duration(); +++ +++ @Override +++ protected long getTestTimeout() { +++ return duration() * 1200; +++ } +++ +++ /** */ +++ protected abstract void operation(IgniteEx ig); +++ +++ /** */ +++ public void testMemoryLeak() throws Exception { +++ +++ final long end = System.nanoTime() + TimeUnit.SECONDS.toNanos(duration()); +++ +++ int tasksCount = Runtime.getRuntime().availableProcessors() * 4; +++ +++ IgniteCompute compute = grid(0).compute().withAsync(); +++ +++ ComputeTaskFuture[] futs = new ComputeTaskFuture[tasksCount]; +++ +++ for (int i = 0; i < tasksCount; i++) { +++ compute.run(new IgniteRunnable() { +++ @IgniteInstanceResource +++ private Ignite ig; +++ +++ @Override +++ public void run() { +++ int i = 0; +++ while (System.nanoTime() < end) { +++ operation((IgniteEx) ig); +++ +++ if(i++ == 100) { +++ check((IgniteEx) ig); +++ i = 0; +++ } +++ } +++ } +++ }); +++ +++ futs[i] = compute.future(); +++ } +++ +++ for (ComputeTaskFuture fut : futs) { +++ fut.get(); +++ } +++ } +++ +++ protected void check(IgniteEx ig) {} +++} ++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java ++new file mode 100644 ++index 0000000..4cd74d0 ++--- /dev/null +++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java ++@@ -0,0 +1,85 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++ +++package org.apache.ignite.internal.processors.database; +++ +++import org.apache.ignite.IgniteCache; +++import org.apache.ignite.configuration.IgniteConfiguration; +++import org.apache.ignite.configuration.MemoryConfiguration; +++import org.apache.ignite.internal.IgniteEx; +++ +++import java.util.Random; +++import java.util.concurrent.ThreadLocalRandom; +++ +++/** +++ * +++ */ +++public class IgniteDbMemoryLeakIndexedTest extends IgniteDbMemoryLeakAbstractTest { +++ +++ @Override +++ protected int duration() { +++ return 300; +++ } +++ +++ @Override +++ protected int gridCount() { +++ return 1; +++ } +++ +++ @Override +++ protected void configure(IgniteConfiguration cfg) { +++ cfg.setMetricsLogFrequency(5000); +++ } +++ +++ @Override +++ protected void configure(MemoryConfiguration mCfg) { +++ mCfg.setPageCacheSize(1024 * 1024); +++ } +++ +++ @Override +++ protected boolean indexingEnabled() { +++ return true; +++ } +++ +++ protected void operation(IgniteEx ig){ +++ IgniteCache<Object, Object> cache = ig.cache("non-primitive"); +++ Random rnd = ThreadLocalRandom.current(); +++ +++ for (int i = 0; i < 1000; i++) { +++ DbKey key = new DbKey(rnd.nextInt(200_000)); +++ +++ DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500)); +++ +++ switch (rnd.nextInt(3)) { +++ case 0: +++ cache.getAndPut(key, v0); +++ case 1: +++ cache.get(key); +++ break; +++ case 2: +++ cache.getAndRemove(key); +++ } +++ } +++ } +++ +++ @Override +++ protected void check(IgniteEx ig) { +++ long pages = ig.context().cache().context().database().pageMemory().loadedPages(); +++ +++ assertTrue(pages < 19100); +++ } +++} ++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java ++new file mode 100644 ++index 0000000..a4d88e1 ++--- /dev/null +++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java ++@@ -0,0 +1,95 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++ +++package org.apache.ignite.internal.processors.database; +++ +++import org.apache.ignite.IgniteCache; +++import org.apache.ignite.configuration.IgniteConfiguration; +++import org.apache.ignite.configuration.MemoryConfiguration; +++import org.apache.ignite.internal.IgniteEx; +++ +++import java.util.Random; +++import java.util.concurrent.ThreadLocalRandom; +++ +++/** +++ * +++ */ +++public class IgniteDbMemoryLeakLargeObjectsTest extends IgniteDbMemoryLeakAbstractTest { +++ +++ private final static int[] ARRAY; +++ static { +++ ARRAY = new int[1024]; +++ Random rnd = new Random(); +++ for (int i = 0; i < ARRAY.length; i++) { +++ ARRAY[i] = rnd.nextInt(); +++ } +++ +++ } +++ +++ @Override +++ protected int duration() { +++ return 300; +++ } +++ +++ @Override +++ protected int gridCount() { +++ return 1; +++ } +++ +++ @Override +++ protected void configure(IgniteConfiguration cfg) { +++ cfg.setMetricsLogFrequency(5000); +++ } +++ +++ @Override +++ protected void configure(MemoryConfiguration mCfg) { +++ mCfg.setPageCacheSize(60 * 1024 * 1024); +++ } +++ +++ @Override +++ protected boolean indexingEnabled() { +++ return false; +++ } +++ +++ protected void operation(IgniteEx ig){ +++ IgniteCache<Object, Object> cache = ig.cache("large"); +++ Random rnd = ThreadLocalRandom.current(); +++ +++ for (int i = 0; i < 1000; i++) { +++ LargeDbKey key = new LargeDbKey(rnd.nextInt(10_000), 1024); +++ +++ LargeDbValue v0 = new LargeDbValue("test-value-1-" + rnd.nextInt(200), "test-value-2-" + rnd.nextInt(200), ARRAY); +++ +++ switch (rnd.nextInt(3)) { +++ case 0: +++ cache.getAndPut(key, v0); +++ case 1: +++ cache.get(key); +++ break; +++ case 2: +++ cache.getAndRemove(key); +++ } +++ } +++ } +++ +++ @Override +++ protected void check(IgniteEx ig) { +++ long pages = ig.context().cache().context().database().pageMemory().loadedPages(); +++ +++ assertTrue(pages < 50000); +++ } +++} ++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java ++new file mode 100644 ++index 0000000..bfa4aa9 ++--- /dev/null +++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java ++@@ -0,0 +1,90 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++ +++package org.apache.ignite.internal.processors.database; +++ +++import org.apache.ignite.IgniteCache; +++import org.apache.ignite.configuration.IgniteConfiguration; +++import org.apache.ignite.configuration.MemoryConfiguration; +++import org.apache.ignite.internal.IgniteEx; +++ +++import java.util.Random; +++import java.util.concurrent.ThreadLocalRandom; +++ +++/** +++ * +++ */ +++public class IgniteDbMemoryLeakLargePagesTest extends IgniteDbMemoryLeakAbstractTest { +++ +++ @Override +++ protected int duration() { +++ return 300; +++ } +++ +++ @Override +++ protected int gridCount() { +++ return 1; +++ } +++ +++ @Override +++ protected void configure(IgniteConfiguration cfg) { +++ cfg.setMetricsLogFrequency(5000); +++ } +++ +++ @Override +++ protected void configure(MemoryConfiguration mCfg) { +++ mCfg.setPageCacheSize(100 * 1024 * 1024); +++ } +++ +++ @Override +++ protected boolean indexingEnabled() { +++ return false; +++ } +++ +++ @Override +++ protected boolean isLargePage() { +++ return true; +++ } +++ +++ protected void operation(IgniteEx ig){ +++ IgniteCache<Object, Object> cache = ig.cache("non-primitive"); +++ Random rnd = ThreadLocalRandom.current(); +++ +++ for (int i = 0; i < 1000; i++) { +++ DbKey key = new DbKey(rnd.nextInt(200_000)); +++ +++ DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500)); +++ +++ switch (rnd.nextInt(3)) { +++ case 0: +++ cache.getAndPut(key, v0); +++ case 1: +++ cache.get(key); +++ break; +++ case 2: +++ cache.getAndRemove(key); +++ } +++ } +++ } +++ +++ @Override +++ protected void check(IgniteEx ig) { +++ long pages = ig.context().cache().context().database().pageMemory().loadedPages(); +++ +++ assertTrue(pages < 4600); +++ } +++} ++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java ++new file mode 100644 ++index 0000000..6af4e41 ++--- /dev/null +++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java ++@@ -0,0 +1,85 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++ +++package org.apache.ignite.internal.processors.database; +++ +++import org.apache.ignite.IgniteCache; +++import org.apache.ignite.configuration.IgniteConfiguration; +++import org.apache.ignite.configuration.MemoryConfiguration; +++import org.apache.ignite.internal.IgniteEx; +++ +++import java.util.Random; +++import java.util.concurrent.ThreadLocalRandom; +++ +++/** +++ * +++ */ +++public class IgniteDbMemoryLeakTest extends IgniteDbMemoryLeakAbstractTest { +++ +++ @Override +++ protected int duration() { +++ return 300; +++ } +++ +++ @Override +++ protected int gridCount() { +++ return 1; +++ } +++ +++ @Override +++ protected void configure(IgniteConfiguration cfg) { +++ cfg.setMetricsLogFrequency(5000); +++ } +++ +++ @Override +++ protected void configure(MemoryConfiguration mCfg) { +++ mCfg.setPageCacheSize(1024 * 1024); +++ } +++ +++ @Override +++ protected boolean indexingEnabled() { +++ return false; +++ } +++ +++ protected void operation(IgniteEx ig){ +++ IgniteCache<Object, Object> cache = ig.cache("non-primitive"); +++ Random rnd = ThreadLocalRandom.current(); +++ +++ for (int i = 0; i < 1000; i++) { +++ DbKey key = new DbKey(rnd.nextInt(200_000)); +++ +++ DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500)); +++ +++ switch (rnd.nextInt(3)) { +++ case 0: +++ cache.getAndPut(key, v0); +++ case 1: +++ cache.get(key); +++ break; +++ case 2: +++ cache.getAndRemove(key); +++ } +++ } +++ } +++ +++ @Override +++ protected void check(IgniteEx ig) { +++ long pages = ig.context().cache().context().database().pageMemory().loadedPages(); +++ +++ assertTrue(pages < 19100); +++ } +++} ++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java ++new file mode 100644 ++index 0000000..d9e3f34 ++--- /dev/null +++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java ++@@ -0,0 +1,92 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++ +++package org.apache.ignite.internal.processors.database; +++ +++import org.apache.ignite.IgniteCache; +++import org.apache.ignite.configuration.IgniteConfiguration; +++import org.apache.ignite.configuration.MemoryConfiguration; +++import org.apache.ignite.internal.IgniteEx; +++ +++import javax.cache.expiry.CreatedExpiryPolicy; +++import javax.cache.expiry.Duration; +++import javax.cache.expiry.ExpiryPolicy; +++import java.util.Random; +++import java.util.concurrent.ThreadLocalRandom; +++ +++import static java.util.concurrent.TimeUnit.MILLISECONDS; +++ +++/** +++ * +++ */ +++public class IgniteDbMemoryLeakWithExpirationTest extends IgniteDbMemoryLeakAbstractTest { +++ +++ private static final ExpiryPolicy EXPIRY = new CreatedExpiryPolicy(new Duration(MILLISECONDS, 10L)); +++ +++ @Override +++ protected int duration() { +++ return 300; +++ } +++ +++ @Override +++ protected int gridCount() { +++ return 1; +++ } +++ +++ @Override +++ protected void configure(IgniteConfiguration cfg) { +++ cfg.setMetricsLogFrequency(5000); +++ } +++ +++ @Override +++ protected void configure(MemoryConfiguration mCfg) { +++ mCfg.setPageCacheSize(1024 * 1024); +++ } +++ +++ @Override +++ protected boolean indexingEnabled() { +++ return false; +++ } +++ +++ protected void operation(IgniteEx ig) { +++ IgniteCache<Object, Object> cache = ig.cache("non-primitive").withExpiryPolicy(EXPIRY); +++ Random rnd = ThreadLocalRandom.current(); +++ +++ for (int i = 0; i < 1000; i++) { +++ DbKey key = new DbKey(rnd.nextInt(200_000)); +++ +++ DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500)); +++ +++ switch (rnd.nextInt(3)) { +++ case 0: +++ cache.getAndPut(key, v0); +++ case 1: +++ cache.get(key); +++ break; +++ case 2: +++ cache.getAndRemove(key); +++ } +++ } +++ } +++ +++ @Override +++ protected void check(IgniteEx ig) { +++ long pages = ig.context().cache().context().database().pageMemory().loadedPages(); +++ +++ assertTrue(pages < 10000); +++ } +++} ++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java ++index c7a07e3..228a262 100644 ++--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java +++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java ++@@ -17,175 +17,39 @@ ++ ++ package org.apache.ignite.internal.processors.database; ++ ++-import java.io.Serializable; ++-import java.util.Arrays; ++-import java.util.HashMap; ++-import java.util.HashSet; ++-import java.util.LinkedHashMap; ++-import java.util.List; ++-import java.util.Map; ++-import java.util.Random; ++-import java.util.Set; ++-import java.util.UUID; ++-import java.util.concurrent.ThreadLocalRandom; ++-import javax.cache.Cache; ++ import org.apache.ignite.Ignite; ++ import org.apache.ignite.IgniteCache; ++ import org.apache.ignite.IgniteDataStreamer; ++-import org.apache.ignite.cache.CacheAtomicityMode; ++ import org.apache.ignite.cache.CachePeekMode; ++-import org.apache.ignite.cache.CacheRebalanceMode; ++-import org.apache.ignite.cache.CacheWriteSynchronizationMode; ++ import org.apache.ignite.cache.affinity.Affinity; ++-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; ++-import org.apache.ignite.cache.affinity.AffinityFunction; ++ import org.apache.ignite.cache.query.QueryCursor; ++ import org.apache.ignite.cache.query.ScanQuery; ++ import org.apache.ignite.cache.query.SqlFieldsQuery; ++ import org.apache.ignite.cache.query.SqlQuery; ++-import org.apache.ignite.cache.query.annotations.QuerySqlField; ++-import org.apache.ignite.configuration.CacheConfiguration; ++-import org.apache.ignite.configuration.MemoryConfiguration; ++-import org.apache.ignite.configuration.IgniteConfiguration; ++ import org.apache.ignite.internal.IgniteEx; ++ import org.apache.ignite.internal.processors.cache.GridCacheAdapter; ++ import org.apache.ignite.internal.processors.cache.database.tree.BPlusTree; ++ import org.apache.ignite.internal.util.GridRandom; ++ import org.apache.ignite.internal.util.typedef.PA; ++ import org.apache.ignite.internal.util.typedef.X; ++-import org.apache.ignite.internal.util.typedef.internal.S; ++-import org.apache.ignite.internal.util.typedef.internal.U; ++-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; ++-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; ++-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; ++ import org.apache.ignite.testframework.GridTestUtils; ++-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; ++ import org.junit.Assert; ++ +++import javax.cache.Cache; +++import java.util.HashMap; +++import java.util.HashSet; +++import java.util.LinkedHashMap; +++import java.util.List; +++import java.util.Map; +++import java.util.Random; +++import java.util.Set; +++import java.util.UUID; +++import java.util.concurrent.ThreadLocalRandom; +++ ++ /** ++ * ++ */ ++-public abstract class IgniteDbPutGetAbstractTest extends GridCommonAbstractTest { ++- /** */ ++- private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); ++- ++- /** ++- * @return Node count. ++- */ ++- protected abstract int gridCount(); ++- ++- /** ++- * @return {@code True} if indexing is enabled. ++- */ ++- protected abstract boolean indexingEnabled(); ++- ++- /** {@inheritDoc} */ ++- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { ++- IgniteConfiguration cfg = super.getConfiguration(gridName); ++- ++- MemoryConfiguration dbCfg = new MemoryConfiguration(); ++- ++- if (isLargePage()) { ++- dbCfg.setConcurrencyLevel(Runtime.getRuntime().availableProcessors() * 4); ++- ++- dbCfg.setPageSize(16 * 1024); ++- ++- dbCfg.setPageCacheSize(200 * 1024 * 1024); ++- } ++- else { ++- dbCfg.setConcurrencyLevel(Runtime.getRuntime().availableProcessors() * 4); ++- ++- dbCfg.setPageSize(1024); ++- ++- dbCfg.setPageCacheSize(200 * 1024 * 1024); ++- } ++- ++- cfg.setMemoryConfiguration(dbCfg); ++- ++- CacheConfiguration ccfg = new CacheConfiguration(); ++- ++- if (indexingEnabled()) ++- ccfg.setIndexedTypes(Integer.class, DbValue.class); ++- ++- ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); ++- ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); ++- ccfg.setRebalanceMode(CacheRebalanceMode.SYNC); ++- ccfg.setAffinity(new RendezvousAffinityFunction(false, 32)); ++- ++- CacheConfiguration ccfg2 = new CacheConfiguration("non-primitive"); ++- ++- if (indexingEnabled()) ++- ccfg2.setIndexedTypes(DbKey.class, DbValue.class); ++- ++- ccfg2.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); ++- ccfg2.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); ++- ccfg2.setRebalanceMode(CacheRebalanceMode.SYNC); ++- ccfg2.setAffinity(new RendezvousAffinityFunction(false, 32)); ++- ++- CacheConfiguration ccfg3 = new CacheConfiguration("large"); ++- ++- if (indexingEnabled()) ++- ccfg3.setIndexedTypes(Integer.class, LargeDbValue.class); ++- ++- ccfg3.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); ++- ccfg3.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); ++- ccfg3.setRebalanceMode(CacheRebalanceMode.SYNC); ++- ccfg3.setAffinity(new RendezvousAffinityFunction(false, 32)); ++- ++- CacheConfiguration ccfg4 = new CacheConfiguration("tiny"); ++- ++- ccfg4.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); ++- ccfg4.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); ++- ccfg4.setRebalanceMode(CacheRebalanceMode.SYNC); ++- ccfg4.setAffinity(new RendezvousAffinityFunction(false, 32)); ++- ++- final AffinityFunction aff = new RendezvousAffinityFunction(1, null); ++- ++- ccfg4.setAffinity(aff); ++- ++- cfg.setCacheConfiguration(ccfg, ccfg2, ccfg3, ccfg4); ++- ++- TcpDiscoverySpi discoSpi = new TcpDiscoverySpi(); ++- ++- discoSpi.setIpFinder(IP_FINDER); ++- ++- cfg.setDiscoverySpi(discoSpi); ++- cfg.setMarshaller(null); ++- ++- return cfg; ++- } ++- ++- /** {@inheritDoc} */ ++- @Override protected void beforeTest() throws Exception { ++- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false)); ++- ++- long seed = 1464583813940L; // System.currentTimeMillis(); ++- ++- info("Seed: " + seed + "L"); ++- ++- BPlusTree.rnd = new Random(seed); ++- ++- startGrids(gridCount()); ++- ++- awaitPartitionMapExchange(); ++- } ++- ++- /** {@inheritDoc} */ ++- @Override protected void afterTest() throws Exception { ++- BPlusTree.rnd = null; ++- ++- stopAllGrids(); ++- ++- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false)); ++- } ++- ++- /** ++- * @return {@code True} if use large page. ++- */ ++- protected boolean isLargePage() { ++- return false; ++- }; ++- +++public abstract class IgniteDbPutGetAbstractTest extends IgniteDbAbstractTest { ++ /** ++ * ++ */ ++@@ -1349,191 +1213,4 @@ private void checkEmpty(final GridCacheAdapter internalCache, final Object key) ++ ++ assertNull(internalCache.peekEx(key)); ++ } ++- ++- /** ++- * ++- */ ++- private static class DbKey implements Serializable { ++- /** */ ++- private int val; ++- ++- /** ++- * @param val Value. ++- */ ++- private DbKey(int val) { ++- this.val = val; ++- } ++- ++- /** {@inheritDoc} */ ++- @Override public boolean equals(Object o) { ++- if (this == o) ++- return true; ++- ++- if (o == null || !(o instanceof DbKey)) ++- return false; ++- ++- DbKey key = (DbKey)o; ++- ++- return val == key.val; ++- } ++- ++- /** {@inheritDoc} */ ++- @Override public int hashCode() { ++- return val; ++- } ++- } ++- ++- /** ++- * ++- */ ++- private static class LargeDbKey implements Serializable { ++- /** */ ++- private int val; ++- ++- /** */ ++- private byte[] data; ++- ++- /** ++- * @param val Value. ++- * @param size Key payload size. ++- */ ++- private LargeDbKey(int val, int size) { ++- this.val = val; ++- ++- data = new byte[size]; ++- ++- Arrays.fill(data, (byte)val); ++- } ++- ++- /** {@inheritDoc} */ ++- @Override public boolean equals(Object o) { ++- if (this == o) ++- return true; ++- ++- if (o == null || !(o instanceof LargeDbKey)) ++- return false; ++- ++- LargeDbKey key = (LargeDbKey)o; ++- ++- return val == key.val && Arrays.equals(data, key.data); ++- } ++- ++- /** {@inheritDoc} */ ++- @Override public int hashCode() { ++- return val + Arrays.hashCode(data); ++- } ++- } ++- ++- /** ++- * ++- */ ++- private static class DbValue implements Serializable { ++- /** */ ++- @QuerySqlField(index = true) ++- private int iVal; ++- ++- /** */ ++- @QuerySqlField(index = true) ++- private String sVal; ++- ++- /** */ ++- @QuerySqlField ++- private long lVal; ++- ++- /** ++- * @param iVal Integer value. ++- * @param sVal String value. ++- * @param lVal Long value. ++- */ ++- public DbValue(int iVal, String sVal, long lVal) { ++- this.iVal = iVal; ++- this.sVal = sVal; ++- this.lVal = lVal; ++- } ++- ++- /** {@inheritDoc} */ ++- @Override public boolean equals(Object o) { ++- if (this == o) ++- return true; ++- ++- if (o == null || getClass() != o.getClass()) ++- return false; ++- ++- DbValue dbVal = (DbValue)o; ++- ++- return iVal == dbVal.iVal && lVal == dbVal.lVal && ++- !(sVal != null ? !sVal.equals(dbVal.sVal) : dbVal.sVal != null); ++- } ++- ++- /** {@inheritDoc} */ ++- @Override public int hashCode() { ++- int res = iVal; ++- ++- res = 31 * res + (sVal != null ? sVal.hashCode() : 0); ++- res = 31 * res + (int)(lVal ^ (lVal >>> 32)); ++- ++- return res; ++- } ++- ++- /** {@inheritDoc} */ ++- @Override public String toString() { ++- return S.toString(DbValue.class, this); ++- } ++- } ++- ++- /** ++- * ++- */ ++- private static class LargeDbValue { ++- /** */ ++- @QuerySqlField(index = true) ++- private String str1; ++- ++- /** */ ++- @QuerySqlField(index = true) ++- private String str2; ++- ++- /** */ ++- private int[] arr; ++- ++- /** ++- * @param str1 String 1. ++- * @param str2 String 2. ++- * @param arr Big array. ++- */ ++- public LargeDbValue(final String str1, final String str2, final int[] arr) { ++- this.str1 = str1; ++- this.str2 = str2; ++- this.arr = arr; ++- } ++- ++- /** {@inheritDoc} */ ++- @Override public boolean equals(final Object o) { ++- if (this == o) return true; ++- if (o == null || getClass() != o.getClass()) return false; ++- ++- final LargeDbValue that = (LargeDbValue) o; ++- ++- if (str1 != null ? !str1.equals(that.str1) : that.str1 != null) return false; ++- if (str2 != null ? !str2.equals(that.str2) : that.str2 != null) return false; ++- ++- return Arrays.equals(arr, that.arr); ++- ++- } ++- ++- /** {@inheritDoc} */ ++- @Override public int hashCode() { ++- int res = str1 != null ? str1.hashCode() : 0; ++- ++- res = 31 * res + (str2 != null ? str2.hashCode() : 0); ++- res = 31 * res + Arrays.hashCode(arr); ++- ++- return res; ++- } ++- ++- /** {@inheritDoc} */ ++- @Override public String toString() { ++- return S.toString(LargeDbValue.class, this); ++- } ++- } ++ } ++ ++From 8e12097f9094d7f155135ee2f4c9c33f5f7af9aa Mon Sep 17 00:00:00 2001 ++From: sboikov <[email protected]> ++Date: Wed, 15 Feb 2017 15:08:14 +0300 ++Subject: [PATCH 02/41] ignite-4694 review ++ ++--- ++ .../database/IgniteDbMemoryLeakAbstractTest.java | 15 ++++++++------- ++ .../database/IgniteDbMemoryLeakIndexedTest.java | 3 +++ ++ .../database/IgniteDbMemoryLeakLargeObjectsTest.java | 9 +++++---- ++ .../database/IgniteDbMemoryLeakLargePagesTest.java | 2 ++ ++ .../processors/database/IgniteDbMemoryLeakTest.java | 10 +++++++--- ++ .../database/IgniteDbMemoryLeakWithExpirationTest.java | 2 +- ++ 6 files changed, 26 insertions(+), 15 deletions(-) ++ ++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java ++index 6a5d039..fc0e715 100644 ++--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java +++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java ++@@ -27,10 +27,10 @@ ++ import java.util.concurrent.TimeUnit; ++ ++ /** ++- * +++ * TODO: fix javadoc warnings, code style. ++ */ ++ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTest { ++- +++ // TODO: take duration from system property. ++ /** Test duration in seconds*/ ++ protected abstract int duration(); ++ ++@@ -44,16 +44,18 @@ protected long getTestTimeout() { ++ ++ /** */ ++ public void testMemoryLeak() throws Exception { +++ // TODO: take PageMemory max size is the same as we configured. ++ ++ final long end = System.nanoTime() + TimeUnit.SECONDS.toNanos(duration()); ++ ++- int tasksCount = Runtime.getRuntime().availableProcessors() * 4; +++ // TODO: use threads instead of compute or make sure there are enough threads in pool. +++ int tasksCnt = Runtime.getRuntime().availableProcessors() * 4; ++ ++ IgniteCompute compute = grid(0).compute().withAsync(); ++ ++- ComputeTaskFuture[] futs = new ComputeTaskFuture[tasksCount]; +++ ComputeTaskFuture[] futs = new ComputeTaskFuture[tasksCnt]; ++ ++- for (int i = 0; i < tasksCount; i++) { +++ for (int i = 0; i < tasksCnt; i++) { ++ compute.run(new IgniteRunnable() { ++ @IgniteInstanceResource ++ private Ignite ig; ++@@ -75,9 +77,8 @@ public void run() { ++ futs[i] = compute.future(); ++ } ++ ++- for (ComputeTaskFuture fut : futs) { +++ for (ComputeTaskFuture fut : futs) ++ fut.get(); ++- } ++ } ++ ++ protected void check(IgniteEx ig) {} ++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java ++index 4cd74d0..db77131 100644 ++--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java +++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java ++@@ -51,6 +51,7 @@ protected void configure(MemoryConfiguration mCfg) { ++ } ++ ++ @Override +++ // TODO: move test to module ignite-indexing. ++ protected boolean indexingEnabled() { ++ return true; ++ } ++@@ -64,6 +65,8 @@ protected void operation(IgniteEx ig){ ++ ++ DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500)); ++ +++ // TODO: also execute sql queries. +++ ++ switch (rnd.nextInt(3)) { ++ case 0: ++ cache.getAndPut(key, v0); ++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java ++index a4d88e1..2a6c8cd 100644 ++--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java +++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java ++@@ -29,15 +29,16 @@ ++ * ++ */ ++ public class IgniteDbMemoryLeakLargeObjectsTest extends IgniteDbMemoryLeakAbstractTest { ++- +++ /** */ ++ private final static int[] ARRAY; +++ ++ static { ++ ARRAY = new int[1024]; +++ ++ Random rnd = new Random(); ++- for (int i = 0; i < ARRAY.length; i++) { ++- ARRAY[i] = rnd.nextInt(); ++- } ++ +++ for (int i = 0; i < ARRAY.length; i++) +++ ARRAY[i] = rnd.nextInt(); ++ } ++ ++ @Override ++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java ++index bfa4aa9..91c96af 100644 ++--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java +++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java ++@@ -47,6 +47,7 @@ protected void configure(IgniteConfiguration cfg) { ++ ++ @Override ++ protected void configure(MemoryConfiguration mCfg) { +++ // TODO: understand why such overhead with large pages. ++ mCfg.setPageCacheSize(100 * 1024 * 1024); ++ } ++ ++@@ -60,6 +61,7 @@ protected boolean isLargePage() { ++ return true; ++ } ++ +++ // TODO: avoid copy/paste. ++ protected void operation(IgniteEx ig){ ++ IgniteCache<Object, Object> cache = ig.cache("non-primitive"); ++ Random rnd = ThreadLocalRandom.current(); ++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java ++index 6af4e41..2b0ce1e 100644 ++--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java +++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java ++@@ -29,9 +29,8 @@ ++ * ++ */ ++ public class IgniteDbMemoryLeakTest extends IgniteDbMemoryLeakAbstractTest { ++- ++- @Override ++- protected int duration() { +++ /** {@inheritDoc} */ +++ @Override protected int duration() { ++ return 300; ++ } ++ ++@@ -64,12 +63,17 @@ protected void operation(IgniteEx ig){ ++ ++ DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500)); ++ +++ // TODO: also execute scan query. +++ ++ switch (rnd.nextInt(3)) { ++ case 0: ++ cache.getAndPut(key, v0); +++ break; +++ ++ case 1: ++ cache.get(key); ++ break; +++ ++ case 2: ++ cache.getAndRemove(key); ++ } ++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java ++index d9e3f34..95fe8c8 100644 ++--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java +++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java ++@@ -34,7 +34,7 @@ ++ * ++ */ ++ public class IgniteDbMemoryLeakWithExpirationTest extends IgniteDbMemoryLeakAbstractTest { ++- +++ /** */ ++ private static final ExpiryPolicy EXPIRY = new CreatedExpiryPolicy(new Duration(MILLISECONDS, 10L)); ++ ++ @Override ++ ++From e70d990f14288cfc8fe211fa25631016d5708144 Mon Sep 17 00:00:00 2001 ++From: Igor Seliverstov <[email protected]> ++Date: Wed, 15 Feb 2017 18:04:38 +0300 ++Subject: [PATCH 03/41] IGNITE-4694 Add tests to check there are no memory ++ leaks in PageMemory ++ ++--- ++ .../cache/IgniteCacheOffheapManagerImpl.java | 2 +- ++ .../processors/database/IgniteDbAbstractTest.java | 6 + ++ .../database/IgniteDbMemoryLeakAbstractTest.java | 172 ++++++++++++++++----- ++ .../database/IgniteDbMemoryLeakIndexedTest.java | 65 +------- ++ .../IgniteDbMemoryLeakLargeObjectsTest.java | 64 ++------ ++ .../database/IgniteDbMemoryLeakLargePagesTest.java | 67 ++------ ++ .../database/IgniteDbMemoryLeakTest.java | 63 ++------ ++ .../IgniteDbMemoryLeakWithExpirationTest.java | 58 +------ ++ 8 files changed, 182 insertions(+), 315 deletions(-) ++ ++diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java ++index 5df99b6..9becc99 100644 ++--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java +++++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java ++@@ -897,7 +897,7 @@ public CacheDataStoreImpl( ++ */ ++ private boolean canUpdateOldRow(@Nullable CacheDataRow oldRow, DataRow dataRow) ++ throws IgniteCheckedException { ++- if (oldRow == null || indexingEnabled) +++ if (oldRow == null || indexingEnabled || oldRow.expireTime() != dataRow.expireTime()) ++ return false; ++ ++ CacheObjectContext coCtx = cctx.cacheObjectContext(); ++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java ++index 3bc7004..9297cec 100644 ++--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java +++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java ++@@ -130,10 +130,16 @@ ++ return cfg; ++ } ++ +++ /** +++ * @param cfg IgniteConfiguration. +++ */ ++ protected void configure(IgniteConfiguration cfg){ ++ //NOP ++ } ++ +++ /** +++ * @param mCfg MemoryConfiguration. +++ */ ++ protected void configure(MemoryConfiguration mCfg){ ++ //NOP ++ } ++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java ++index fc0e715..bca3af0 100644 ++--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java +++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java ++@@ -17,69 +17,157 @@ ++ ++ package org.apache.ignite.internal.processors.database; ++ ++-import org.apache.ignite.Ignite; ++-import org.apache.ignite.IgniteCompute; ++-import org.apache.ignite.compute.ComputeTaskFuture; ++-import org.apache.ignite.internal.IgniteEx; ++-import org.apache.ignite.lang.IgniteRunnable; ++-import org.apache.ignite.resources.IgniteInstanceResource; ++- +++import java.util.Random; ++ import java.util.concurrent.TimeUnit; +++import org.apache.ignite.IgniteCache; +++import org.apache.ignite.configuration.IgniteConfiguration; +++import org.apache.ignite.configuration.MemoryConfiguration; +++import org.apache.ignite.internal.IgniteEx; +++import org.apache.ignite.internal.util.GridRandom; +++import org.jetbrains.annotations.NotNull; ++ ++ /** ++- * TODO: fix javadoc warnings, code style. +++ * Base class for memory leaks tests. ++ */ ++ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTest { ++- // TODO: take duration from system property. ++- /** Test duration in seconds*/ ++- protected abstract int duration(); ++- ++- @Override ++- protected long getTestTimeout() { ++- return duration() * 1200; ++- } ++ ++ /** */ ++- protected abstract void operation(IgniteEx ig); +++ private volatile Exception ex = null; ++ ++ /** */ ++- public void testMemoryLeak() throws Exception { ++- // TODO: take PageMemory max size is the same as we configured. +++ private static final ThreadLocal<Random> THREAD_LOCAL_RANDOM = new ThreadLocal<>(); ++ ++- final long end = System.nanoTime() + TimeUnit.SECONDS.toNanos(duration()); +++ /** {@inheritDoc} */ +++ @Override protected void configure(IgniteConfiguration cfg) { +++ cfg.setMetricsLogFrequency(5000); +++ } ++ ++- // TODO: use threads instead of compute or make sure there are enough threads in pool. ++- int tasksCnt = Runtime.getRuntime().availableProcessors() * 4; +++ /** {@inheritDoc} */ +++ @Override protected void configure(MemoryConfiguration mCfg) { +++ int concLvl = Runtime.getRuntime().availableProcessors(); ++ ++- IgniteCompute compute = grid(0).compute().withAsync(); +++ mCfg.setConcurrencyLevel(concLvl); +++ mCfg.setPageCacheSize(1024 * 1024 * concLvl); //minimal possible value +++ } ++ ++- ComputeTaskFuture[] futs = new ComputeTaskFuture[tasksCnt]; +++ /** +++ * @return Test duration in seconds. +++ */ +++ protected int duration() { +++ return 300; +++ } +++ +++ /** {@inheritDoc} */ +++ @Override protected int gridCount() { +++ return 1; +++ } +++ +++ /** {@inheritDoc} */ +++ @Override protected boolean indexingEnabled() { +++ return false; +++ } ++ ++- for (int i = 0; i < tasksCnt; i++) { ++- compute.run(new IgniteRunnable() { ++- @IgniteInstanceResource ++- private Ignite ig; +++ /** {@inheritDoc} */ +++ @Override protected long getTestTimeout() { +++ return (duration() + 1) * 1000; +++ } ++ ++- @Override ++- public void run() { ++- int i = 0; ++- while (System.nanoTime() < end) { ++- operation((IgniteEx) ig); +++ /** +++ * @param ig Ignite instance. +++ * @return IgniteCache. +++ */ +++ protected abstract IgniteCache<Object,Object> cache(IgniteEx ig); +++ +++ /** +++ * @return Cache key to perform an operation. +++ */ +++ protected abstract Object key(); +++ +++ /** +++ * @return Cache value to perform an operation. +++ * @param key Cache key to perform an operation. +++ */ +++ protected abstract Object value(Object key); +++ +++ /** +++ * @param cache IgniteCache. +++ */ +++ protected void operation(IgniteCache<Object, Object> cache) { +++ Object key = key(); +++ Object value = value(key); +++ +++ switch (getRandom().nextInt(3)) { +++ case 0: +++ cache.getAndPut(key, value); +++ case 1: +++ cache.get(key); +++ break; +++ case 2: +++ cache.getAndRemove(key); +++ } +++ } ++ ++- if(i++ == 100) { ++- check((IgniteEx) ig); ++- i = 0; ++- } +++ /** +++ * @return Random. +++ */ +++ @NotNull protected static Random getRandom() { +++ Random rnd = THREAD_LOCAL_RANDOM.get(); +++ +++ if(rnd == null){ +++ rnd = new GridRandom(); +++ THREAD_LOCAL_RANDOM.set(rnd); +++ } +++ +++ return rnd; +++ } +++ +++ /** +++ * @throws Exception If failed. +++ */ +++ public void testMemoryLeak() throws Exception { +++ final long end = System.nanoTime() + TimeUnit.SECONDS.toNanos(duration()); +++ +++ final IgniteEx ignite = grid(0); +++ final IgniteCache<Object, Object> cache = cache(ignite); +++ +++ Runnable target = new Runnable() { +++ @Override public void run() { +++ while (ex == null && System.nanoTime() < end) { +++ try { +++ operation(cache); +++ check(ignite); +++ } +++ catch (Exception e) { +++ ex = e; +++ +++ break; ++ } ++ } ++- }); +++ } +++ }; +++ +++ Thread[] threads = new Thread[Runtime.getRuntime().availableProcessors()]; ++ ++- futs[i] = compute.future(); +++ for (int i = 0; i < threads.length; i++) { +++ threads[i] = new Thread(target); +++ threads[i].start(); ++ } ++ ++- for (ComputeTaskFuture fut : futs) ++- fut.get(); +++ for (Thread thread : threads) { +++ thread.join(); +++ } +++ +++ if(ex != null){ +++ throw ex; +++ } ++ } ++ ++- protected void check(IgniteEx ig) {} +++ /** +++ * Callback to check the current state +++ * +++ * @param ig Ignite instance +++ * @throws Exception If failed. +++ */ +++ protected void check(IgniteEx ig) throws Exception { +++ } ++ } ++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java ++index db77131..acc6c2f 100644 ++--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java +++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java ++@@ -17,72 +17,13 @@ ++ ++ package org.apache.ignite.internal.processors.database; ++ ++-import org.apache.ignite.IgniteCache; ++-import org.apache.ignite.configuration.IgniteConfiguration; ++-import org.apache.ignite.configuration.MemoryConfiguration; ++-import org.apache.ignite.internal.IgniteEx; ++- ++-import java.util.Random; ++-import java.util.concurrent.ThreadLocalRandom; ++- ++ /** ++ * ++ */ ++-public class IgniteDbMemoryLeakIndexedTest extends IgniteDbMemoryLeakAbstractTest { ++- ++- @Override ++- protected int duration() { ++- return 300; ++- } ++- ++- @Override ++- protected int gridCount() { ++- return 1; ++- } ++- ++- @Override ++- protected void configure(IgniteConfiguration cfg) { ++- cfg.setMetricsLogFrequency(5000); ++- } ++- ++- @Override ++- protected void configure(MemoryConfiguration mCfg) { ++- mCfg.setPageCacheSize(1024 * 1024); ++- } +++public class IgniteDbMemoryLeakIndexedTest extends IgniteDbMemoryLeakTest { ++ ++- @Override ++- // TODO: move test to module ignite-indexing. ++- protected boolean indexingEnabled() { +++ /** {@inheritDoc} */ +++ @Override protected boolean indexingEnabled() { ++ return true; ++ } ++- ++- protected void operation(IgniteEx ig){ ++- IgniteCache<Object, Object> cache = ig.cache("non-primitive"); ++- Random rnd = ThreadLocalRandom.current(); ++- ++- for (int i = 0; i < 1000; i++) { ++- DbKey key = new DbKey(rnd.nextInt(200_000)); ++- ++- DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500)); ++- ++- // TODO: also execute sql queries. ++- ++- switch (rnd.nextInt(3)) { ++- case 0: ++- cache.getAndPut(key, v0); ++- case 1: ++- cache.get(key); ++- break; ++- case 2: ++- cache.getAndRemove(key); ++- } ++- } ++- } ++- ++- @Override ++- protected void check(IgniteEx ig) { ++- long pages = ig.context().cache().context().database().pageMemory().loadedPages(); ++- ++- assertTrue(pages < 19100); ++- } ++ } ++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java ++index 2a6c8cd..8943743 100644 ++--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java +++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java ++@@ -18,13 +18,8 @@ ++ package org.apache.ignite.internal.processors.database; ++ ++ import org.apache.ignite.IgniteCache; ++-import org.apache.ignite.configuration.IgniteConfiguration; ++-import org.apache.ignite.configuration.MemoryConfiguration; ++ import org.apache.ignite.internal.IgniteEx; ++ ++-import java.util.Random; ++-import java.util.concurrent.ThreadLocalRandom; ++- ++ /** ++ * ++ */ ++@@ -35,62 +30,29 @@ ++ static { ++ ARRAY = new int[1024]; ++ ++- Random rnd = new Random(); ++- ++ for (int i = 0; i < ARRAY.length; i++) ++- ARRAY[i] = rnd.nextInt(); ++- } ++- ++- @Override ++- protected int duration() { ++- return 300; ++- } ++- ++- @Override ++- protected int gridCount() { ++- return 1; +++ ARRAY[i] = getRandom().nextInt(); ++ } ++ ++- @Override ++- protected void configure(IgniteConfiguration cfg) { ++- cfg.setMetricsLogFrequency(5000); +++ /** {@inheritDoc} */ +++ @Override protected IgniteCache<Object, Object> cache(IgniteEx ig) { +++ return ig.cache("non-primitive"); ++ } ++ ++- @Override ++- protected void configure(MemoryConfiguration mCfg) { ++- mCfg.setPageCacheSize(60 * 1024 * 1024); +++ /** {@inheritDoc} */ +++ @Override protected Object key() { +++ return new DbKey(getRandom().nextInt(200_000)); ++ } ++ ++- @Override ++- protected boolean indexingEnabled() { ++- return false; ++- } ++- ++- protected void operation(IgniteEx ig){ ++- IgniteCache<Object, Object> cache = ig.cache("large"); ++- Random rnd = ThreadLocalRandom.current(); ++- ++- for (int i = 0; i < 1000; i++) { ++- LargeDbKey key = new LargeDbKey(rnd.nextInt(10_000), 1024); ++- ++- LargeDbValue v0 = new LargeDbValue("test-value-1-" + rnd.nextInt(200), "test-value-2-" + rnd.nextInt(200), ARRAY); ++- ++- switch (rnd.nextInt(3)) { ++- case 0: ++- cache.getAndPut(key, v0); ++- case 1: ++- cache.get(key); ++- break; ++- case 2: ++- cache.getAndRemove(key); ++- } ++- } +++ /** {@inheritDoc} */ +++ @Override protected Object value(Object key) { +++ return new DbValue(((DbKey)key).val, "test-value-" + getRandom().nextInt(200), getRandom().nextInt(500)); ++ } ++ ++- @Override ++- protected void check(IgniteEx ig) { +++ /** {@inheritDoc} */ +++ @Override protected void check(IgniteEx ig) { ++ long pages = ig.context().cache().context().database().pageMemory().loadedPages(); ++ ++- assertTrue(pages < 50000); +++ assertTrue(pages < 20000); ++ } ++ } ++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java ++index 91c96af..8e4d0b4 100644 ++--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java +++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java ++@@ -17,76 +17,31 @@ ++ ++ package org.apache.ignite.internal.processors.database; ++ ++-import org.apache.ignite.IgniteCache; ++-import org.apache.ignite.configuration.IgniteConfiguration; ++ import org.apache.ignite.configuration.MemoryConfiguration; ++ import org.apache.ignite.internal.IgniteEx; ++ ++-import java.util.Random; ++-import java.util.concurrent.ThreadLocalRandom; ++- ++ /** ++ * ++ */ ++-public class IgniteDbMemoryLeakLargePagesTest extends IgniteDbMemoryLeakAbstractTest { ++- ++- @Override ++- protected int duration() { ++- return 300; ++- } +++public class IgniteDbMemoryLeakLargePagesTest extends IgniteDbMemoryLeakTest { ++ ++- @Override ++- protected int gridCount() { ++- return 1; ++- } +++ /** {@inheritDoc} */ +++ @Override protected void configure(MemoryConfiguration mCfg) { +++ int concLvl = Runtime.getRuntime().availableProcessors(); +++ mCfg.setConcurrencyLevel(concLvl); +++ mCfg.setPageCacheSize(1024 * 1024 * concLvl * 16); ++ ++- @Override ++- protected void configure(IgniteConfiguration cfg) { ++- cfg.setMetricsLogFrequency(5000); ++ } ++ ++- @Override ++- protected void configure(MemoryConfiguration mCfg) { ++- // TODO: understand why such overhead with large pages. ++- mCfg.setPageCacheSize(100 * 1024 * 1024); ++- } ++- ++- @Override ++- protected boolean indexingEnabled() { ++- return false; ++- } ++- ++- @Override ++- protected boolean isLargePage() { +++ /** {@inheritDoc} */ +++ @Override protected boolean isLargePage() { ++ return true; ++ } ++ ++- // TODO: avoid copy/paste. ++- protected void operation(IgniteEx ig){ ++- IgniteCache<Object, Object> cache = ig.cache("non-primitive"); ++- Random rnd = ThreadLocalRandom.current(); ++- ++- for (int i = 0; i < 1000; i++) { ++- DbKey key = new DbKey(rnd.nextInt(200_000)); ++- ++- DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500)); ++- ++- switch (rnd.nextInt(3)) { ++- case 0: ++- cache.getAndPut(key, v0); ++- case 1: ++- cache.get(key); ++- break; ++- case 2: ++- cache.getAndRemove(key); ++- } ++- } ++- } ++- ++- @Override ++- protected void check(IgniteEx ig) { +++ /** {@inheritDoc} */ +++ @Override protected void check(IgniteEx ig) { ++ long pages = ig.context().cache().context().database().pageMemory().loadedPages(); ++ ++- assertTrue(pages < 4600); +++ assertTrue(pages < 4000); ++ } ++ } ++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java ++index 2b0ce1e..81d831b 100644 ++--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java +++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java ++@@ -18,72 +18,31 @@ ++ package org.apache.ignite.internal.processors.database; ++ ++ import org.apache.ignite.IgniteCache; ++-import org.apache.ignite.configuration.IgniteConfiguration; ++-import org.apache.ignite.configuration.MemoryConfiguration; ++ import org.apache.ignite.internal.IgniteEx; ++ ++-import java.util.Random; ++-import java.util.concurrent.ThreadLocalRandom; ++- ++ /** ++ * ++ */ ++ public class IgniteDbMemoryLeakTest extends IgniteDbMemoryLeakAbstractTest { ++ /** {@inheritDoc} */ ++- @Override protected int duration() { ++- return 300; ++- } ++- ++- @Override ++- protected int gridCount() { ++- return 1; ++- } ++- ++- @Override ++- protected void configure(IgniteConfiguration cfg) { ++- cfg.setMetricsLogFrequency(5000); ++- } ++- ++- @Override ++- protected void configure(MemoryConfiguration mCfg) { ++- mCfg.setPageCacheSize(1024 * 1024); +++ @Override protected IgniteCache<Object, Object> cache(IgniteEx ig) { +++ return ig.cache("non-primitive"); ++ } ++ ++- @Override ++- protected boolean indexingEnabled() { ++- return false; +++ /** {@inheritDoc} */ +++ @Override protected Object key() { +++ return new DbKey(getRandom().nextInt(200_000)); ++ } ++ ++- protected void operation(IgniteEx ig){ ++- IgniteCache<Object, Object> cache = ig.cache("non-primitive"); ++- Random rnd = ThreadLocalRandom.current(); ++- ++- for (int i = 0; i < 1000; i++) { ++- DbKey key = new DbKey(rnd.nextInt(200_000)); ++- ++- DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500)); ++- ++- // TODO: also execute scan query. ++- ++- switch (rnd.nextInt(3)) { ++- case 0: ++- cache.getAndPut(key, v0); ++- break; ++- ++- case 1: ++- cache.get(key); ++- break; ++- ++- case 2: ++- cache.getAndRemove(key); ++- } ++- } +++ /** {@inheritDoc} */ +++ @Override protected Object value(Object key) { +++ return new DbValue(((DbKey)key).val, "test-value-" + getRandom().nextInt(200), getRandom().nextInt(500)); ++ } ++ ++- @Override ++- protected void check(IgniteEx ig) { +++ /** {@inheritDoc} */ +++ @Override protected void check(IgniteEx ig) { ++ long pages = ig.context().cache().context().database().pageMemory().loadedPages(); ++ ++- assertTrue(pages < 19100); +++ assertTrue(pages < 20000); ++ } ++ } ++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java ++index 95fe8c8..a31ffb4 100644 ++--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java +++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java ++@@ -18,75 +18,31 @@ ++ package org.apache.ignite.internal.processors.database; ++ ++ import org.apache.ignite.IgniteCache; ++-import org.apache.ignite.configuration.IgniteConfiguration; ++ import org.apache.ignite.configuration.MemoryConfiguration; ++ import org.apache.ignite.internal.IgniteEx; ++ ++ import javax.cache.expiry.CreatedExpiryPolicy; ++ import javax.cache.expiry.Duration; ++ import javax.cache.expiry.ExpiryPolicy; ++-import java.util.Random; ++-import java.util.concurrent.ThreadLocalRandom; ++ ++ import static java.util.concurrent.TimeUnit.MILLISECONDS; ++ ++ /** ++ * ++ */ ++-public class IgniteDbMemoryLeakWithExpirationTest extends IgniteDbMemoryLeakAbstractTest { +++public class IgniteDbMemoryLeakWithExpirationTest extends IgniteDbMemoryLeakTest { ++ /** */ ++ private static final ExpiryPolicy EXPIRY = new CreatedExpiryPolicy(new Duration(MILLISECONDS, 10L)); ++ ++- @Override ++- protected int duration() { ++- return 300; +++ /** {@inheritDoc} */ +++ @Override protected IgniteCache<Object, Object> cache(IgniteEx ig) { +++ return ig.cache("non-primitive").withExpiryPolicy(EXPIRY); ++ } ++ ++- @Override ++- protected int gridCount() { ++- return 1; ++- } ++- ++- @Override ++- protected void configure(IgniteConfiguration cfg) { ++- cfg.setMetricsLogFrequency(5000); ++- } ++- ++- @Override ++- protected void configure(MemoryConfiguration mCfg) { ++- mCfg.setPageCacheSize(1024 * 1024); ++- } ++- ++- @Override ++- protected boolean indexingEnabled() { ++- return false; ++- } ++- ++- protected void operation(IgniteEx ig) { ++- IgniteCache<Object, Object> cache = ig.cache("non-primitive").withExpiryPolicy(EXPIRY); ++- Random rnd = ThreadLocalRandom.current(); ++- ++- for (int i = 0; i < 1000; i++) { ++- DbKey key = new DbKey(rnd.nextInt(200_000)); ++- ++- DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500)); ++- ++- switch (rnd.nextInt(3)) { ++- case 0: ++- cache.getAndPut(key, v0); ++- case 1: ++- cache.get(key); ++- break; ++- case 2: ++- cache.getAndRemove(key); ++- } ++- } ++- } ++- ++- @Override ++- protected void check(IgniteEx ig) { +++ /** {@inheritDoc} */ +++ @Override protected void check(IgniteEx ig) { ++ long pages = ig.context().cache().context().database().pageMemory().loadedPages(); ++ ++- assertTrue(pages < 10000); +++ assertTrue(pages < 7000); ++ } ++ } ++ ++From 84c03e0c522abc90b2d91e514138eac08388abd2 Mon Sep 17 00:00:00 2001 ++From: Igor Seliverstov <[email protected]> ++Date: Thu, 16 Feb 2017 13:41:51 +0300 ++Subject: [PATCH 04/41] IGNITE-4694 Add tests to check there are no memory ++ leaks in PageMemory (pending) ++ ++--- ++ .../database/IgniteDbMemoryLeakAbstractTest.java | 118 +++++++++++++++++---- ++ .../database/IgniteDbMemoryLeakIndexedTest.java | 29 ----- ++ .../IgniteDbMemoryLeakLargeObjectsTest.java | 16 +-- ++ .../database/IgniteDbMemoryLeakLargePagesTest.java | 13 +-- ++ .../database/IgniteDbMemoryLeakTest.java | 14 ++- ++ .../IgniteDbMemoryLeakWithExpirationTest.java | 7 +- ++ .../database/IgniteDbMemoryLeakIndexedTest.java | 42 ++++++++ ++ 7 files changed, 165 insertions(+), 74 deletions(-) ++ delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java ++ create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java ++ ++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java ++index bca3af0..819405e 100644 ++--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java +++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java ++@@ -30,13 +30,37 @@ ++ * Base class for memory leaks tests. ++ */ ++ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTest { ++- +++ /** */ +++ @SuppressWarnings("WeakerAccess") protected static final int CONCURRENCY_LEVEL = 8; ++ /** */ ++ private volatile Exception ex = null; ++ ++ /** */ +++ private long warmUpEndTime; +++ +++ /** */ +++ private long endTime; +++ +++ /** */ +++ private long loadedPages = 0; +++ +++ /** */ +++ private long delta = 0; +++ +++ /** */ +++ private long probeCnt = 0; +++ +++ /** */ ++ private static final ThreadLocal<Random> THREAD_LOCAL_RANDOM = new ThreadLocal<>(); ++ +++ @Override protected void beforeTest() throws Exception { +++ super.beforeTest(); +++ +++ long startTime = System.nanoTime(); +++ warmUpEndTime = startTime + TimeUnit.SECONDS.toNanos(warmUp()); +++ endTime = warmUpEndTime + TimeUnit.SECONDS.toNanos(duration()); +++ } +++ ++ /** {@inheritDoc} */ ++ @Override protected void configure(IgniteConfiguration cfg) { ++ cfg.setMetricsLogFrequency(5000); ++@@ -44,10 +68,7 @@ ++ ++ /** {@inheritDoc} */ ++ @Override protected void configure(MemoryConfiguration mCfg) { ++- int concLvl = Runtime.getRuntime().availableProcessors(); ++- ++- mCfg.setConcurrencyLevel(concLvl); ++- mCfg.setPageCacheSize(1024 * 1024 * concLvl); //minimal possible value +++ mCfg.setConcurrencyLevel(CONCURRENCY_LEVEL); ++ } ++ ++ /** ++@@ -57,6 +78,13 @@ protected int duration() { ++ return 300; ++ } ++ +++ /** +++ * @return Warm up duration. +++ */ +++ @SuppressWarnings("WeakerAccess") protected int warmUp() { +++ return 300; +++ } +++ ++ /** {@inheritDoc} */ ++ @Override protected int gridCount() { ++ return 1; ++@@ -69,14 +97,14 @@ protected int duration() { ++ ++ /** {@inheritDoc} */ ++ @Override protected long getTestTimeout() { ++- return (duration() + 1) * 1000; +++ return (warmUp() + duration() + 1) * 1000; // One extra second to stop all threads ++ } ++ ++ /** ++ * @param ig Ignite instance. ++ * @return IgniteCache. ++ */ ++- protected abstract IgniteCache<Object,Object> cache(IgniteEx ig); +++ protected abstract IgniteCache<Object, Object> cache(IgniteEx ig); ++ ++ /** ++ * @return Cache key to perform an operation. ++@@ -84,8 +112,8 @@ protected int duration() { ++ protected abstract Object key(); ++ ++ /** ++- * @return Cache value to perform an operation. ++ * @param key Cache key to perform an operation. +++ * @return Cache value to perform an operation. ++ */ ++ protected abstract Object value(Object key); ++ ++@@ -99,8 +127,11 @@ protected void operation(IgniteCache<Object, Object> cache) { ++ switch (getRandom().nextInt(3)) { ++ case 0: ++ cache.getAndPut(key, value); +++ +++ break; ++ case 1: ++ cache.get(key); +++ ++ break; ++ case 2: ++ cache.getAndRemove(key); ++@@ -113,7 +144,7 @@ protected void operation(IgniteCache<Object, Object> cache) { ++ @NotNull protected static Random getRandom() { ++ Random rnd = THREAD_LOCAL_RANDOM.get(); ++ ++- if(rnd == null){ +++ if (rnd == null) { ++ rnd = new GridRandom(); ++ THREAD_LOCAL_RANDOM.set(rnd); ++ } ++@@ -125,49 +156,96 @@ protected void operation(IgniteCache<Object, Object> cache) { ++ * @throws Exception If failed. ++ */ ++ public void testMemoryLeak() throws Exception { ++- final long end = System.nanoTime() + TimeUnit.SECONDS.toNanos(duration()); ++ ++ final IgniteEx ignite = grid(0); ++ final IgniteCache<Object, Object> cache = cache(ignite); ++ ++ Runnable target = new Runnable() { ++ @Override public void run() { ++- while (ex == null && System.nanoTime() < end) { +++ while (ex == null && System.nanoTime() < endTime) { ++ try { ++ operation(cache); ++- check(ignite); ++ } ++ catch (Exception e) { ++ ex = e; ++- ++ break; ++ } ++ } ++ } ++ }; ++ ++- Thread[] threads = new Thread[Runtime.getRuntime().availableProcessors()]; +++ Thread[] threads = new Thread[CONCURRENCY_LEVEL]; +++ +++ info("Warming up is started."); ++ ++ for (int i = 0; i < threads.length; i++) { ++ threads[i] = new Thread(target); ++ threads[i].start(); ++ } ++ ++- for (Thread thread : threads) { ++- thread.join(); +++ Thread.sleep(TimeUnit.NANOSECONDS.toMillis(warmUpEndTime - System.nanoTime())); +++ +++ info("Warming up is ended."); +++ +++ while (System.nanoTime() < endTime) { +++ try { +++ check(ignite); +++ } +++ catch (Exception e) { +++ ex = e; +++ +++ break; +++ } +++ +++ Thread.sleep(TimeUnit.SECONDS.toMillis(5)); ++ } ++ ++- if(ex != null){ +++ for (Thread thread : threads) +++ thread.join(); +++ +++ if (ex != null) ++ throw ex; ++- } ++ } ++ ++ /** ++- * Callback to check the current state +++ * Callback to check the current state. ++ * ++- * @param ig Ignite instance +++ * @param ig Ignite instance. ++ * @throws Exception If failed. ++ */ ++ protected void check(IgniteEx ig) throws Exception { +++ long pagesActual = ig.context().cache().context().database().pageMemory().loadedPages(); +++ long pagesMax = pagesMax(); +++ +++ assertTrue( +++ "Maximal allowed pages number is exceeded. [allowed=" + pagesMax + "; actual= " + pagesActual + "]", +++ pagesActual <= pagesMax); +++ +++ if (loadedPages > 0) { +++ delta += pagesActual - loadedPages; +++ int allowedDelta = pagesDelta(); +++ +++ if(probeCnt++ > 12) { // we need some statistic first. Minimal statistic is taken for a minute. +++ long actualDelta = delta / probeCnt; +++ +++ assertTrue( +++ "Average growth pages in the number is more than expected. [allowed=" + allowedDelta + "; actual=" + actualDelta + "]", +++ actualDelta <= allowedDelta); +++ } +++ } +++ +++ loadedPages = pagesActual; +++ } +++ +++ /** +++ * @return Maximal allowed pages number. +++ */ +++ protected abstract long pagesMax(); +++ +++ /** +++ * @return Expected average number of pages, on which their total number can grow per 5 seconds. +++ */ +++ @SuppressWarnings("WeakerAccess") protected int pagesDelta() { +++ return 5; ++ } ++ } ++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java ++deleted file mode 100644 ++index acc6c2f..0000000 ++--- a/modules/core/src/test/java/org/apache/ignite/internal/processors
<TRUNCATED> http://git-wip-us.apache.org/repos/asf/ignite/blob/81ae2d83/modules/clients/src/test/java/org/apache/ignite/internal/client/integration/ClientAbstractSelfTest.java ----------------------------------------------------------------------
