http://git-wip-us.apache.org/repos/asf/ignite/blob/2c4ef9ee/leak.patch ---------------------------------------------------------------------- diff --git a/leak.patch b/leak.patch deleted file mode 100644 index f618073..0000000 --- a/leak.patch +++ /dev/null @@ -1,10521 +0,0 @@ -From 3c9e3786a21ec7c8919de9c29bc04d2e3561846c Mon Sep 17 00:00:00 2001 -From: Igor Seliverstov <[email protected]> -Date: Wed, 15 Feb 2017 13:41:08 +0300 -Subject: [PATCH 01/41] IGNITE-4694 Add tests to check there are no memory - leaks in PageMemory - ---- - .../processors/database/IgniteDbAbstractTest.java | 360 +++++++++++++++++++++ - .../database/IgniteDbMemoryLeakAbstractTest.java | 84 +++++ - .../database/IgniteDbMemoryLeakIndexedTest.java | 85 +++++ - .../IgniteDbMemoryLeakLargeObjectsTest.java | 95 ++++++ - .../database/IgniteDbMemoryLeakLargePagesTest.java | 90 ++++++ - .../database/IgniteDbMemoryLeakTest.java | 85 +++++ - .../IgniteDbMemoryLeakWithExpirationTest.java | 92 ++++++ - .../database/IgniteDbPutGetAbstractTest.java | 347 +------------------- - 8 files changed, 903 insertions(+), 335 deletions(-) - create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java - create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java - create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java - create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java - create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java - create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java - create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java - -diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java -new file mode 100644 -index 0000000..3bc7004 ---- /dev/null -+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java -@@ -0,0 +1,360 @@ -+/* -+ * Licensed to the Apache Software Foundation (ASF) under one or more -+ * contributor license agreements. See the NOTICE file distributed with -+ * this work for additional information regarding copyright ownership. -+ * The ASF licenses this file to You under the Apache License, Version 2.0 -+ * (the "License"); you may not use this file except in compliance with -+ * the License. You may obtain a copy of the License at -+ * -+ * http://www.apache.org/licenses/LICENSE-2.0 -+ * -+ * Unless required by applicable law or agreed to in writing, software -+ * distributed under the License is distributed on an "AS IS" BASIS, -+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+ * See the License for the specific language governing permissions and -+ * limitations under the License. -+ */ -+ -+package org.apache.ignite.internal.processors.database; -+ -+import org.apache.ignite.cache.CacheAtomicityMode; -+import org.apache.ignite.cache.CacheRebalanceMode; -+import org.apache.ignite.cache.CacheWriteSynchronizationMode; -+import org.apache.ignite.cache.affinity.AffinityFunction; -+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; -+import org.apache.ignite.cache.query.annotations.QuerySqlField; -+import org.apache.ignite.configuration.CacheConfiguration; -+import org.apache.ignite.configuration.IgniteConfiguration; -+import org.apache.ignite.configuration.MemoryConfiguration; -+import org.apache.ignite.internal.processors.cache.database.tree.BPlusTree; -+import org.apache.ignite.internal.util.typedef.internal.S; -+import org.apache.ignite.internal.util.typedef.internal.U; -+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; -+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; -+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; -+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; -+ -+import java.io.Serializable; -+import java.util.Arrays; -+import java.util.Random; -+ -+/** -+ * -+ */ -+public abstract class IgniteDbAbstractTest extends GridCommonAbstractTest { -+ /** */ -+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); -+ -+ /** -+ * @return Node count. -+ */ -+ protected abstract int gridCount(); -+ -+ /** -+ * @return {@code True} if indexing is enabled. -+ */ -+ protected abstract boolean indexingEnabled(); -+ -+ /** {@inheritDoc} */ -+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { -+ IgniteConfiguration cfg = super.getConfiguration(gridName); -+ -+ MemoryConfiguration dbCfg = new MemoryConfiguration(); -+ -+ dbCfg.setConcurrencyLevel(Runtime.getRuntime().availableProcessors() * 4); -+ -+ if (isLargePage()) -+ dbCfg.setPageSize(16 * 1024); -+ else -+ dbCfg.setPageSize(1024); -+ -+ -+ dbCfg.setPageCacheSize(200 * 1024 * 1024); -+ -+ configure(dbCfg); -+ -+ cfg.setMemoryConfiguration(dbCfg); -+ -+ CacheConfiguration ccfg = new CacheConfiguration(); -+ -+ if (indexingEnabled()) -+ ccfg.setIndexedTypes(Integer.class, DbValue.class); -+ -+ ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); -+ ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); -+ ccfg.setRebalanceMode(CacheRebalanceMode.SYNC); -+ ccfg.setAffinity(new RendezvousAffinityFunction(false, 32)); -+ -+ CacheConfiguration ccfg2 = new CacheConfiguration("non-primitive"); -+ -+ if (indexingEnabled()) -+ ccfg2.setIndexedTypes(DbKey.class, DbValue.class); -+ -+ ccfg2.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); -+ ccfg2.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); -+ ccfg2.setRebalanceMode(CacheRebalanceMode.SYNC); -+ ccfg2.setAffinity(new RendezvousAffinityFunction(false, 32)); -+ -+ CacheConfiguration ccfg3 = new CacheConfiguration("large"); -+ -+ if (indexingEnabled()) -+ ccfg3.setIndexedTypes(Integer.class, LargeDbValue.class); -+ -+ ccfg3.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); -+ ccfg3.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); -+ ccfg3.setRebalanceMode(CacheRebalanceMode.SYNC); -+ ccfg3.setAffinity(new RendezvousAffinityFunction(false, 32)); -+ -+ CacheConfiguration ccfg4 = new CacheConfiguration("tiny"); -+ -+ ccfg4.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); -+ ccfg4.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); -+ ccfg4.setRebalanceMode(CacheRebalanceMode.SYNC); -+ ccfg4.setAffinity(new RendezvousAffinityFunction(false, 32)); -+ -+ final AffinityFunction aff = new RendezvousAffinityFunction(1, null); -+ -+ ccfg4.setAffinity(aff); -+ -+ cfg.setCacheConfiguration(ccfg, ccfg2, ccfg3, ccfg4); -+ -+ TcpDiscoverySpi discoSpi = new TcpDiscoverySpi(); -+ -+ discoSpi.setIpFinder(IP_FINDER); -+ -+ cfg.setDiscoverySpi(discoSpi); -+ cfg.setMarshaller(null); -+ -+ configure(cfg); -+ -+ return cfg; -+ } -+ -+ protected void configure(IgniteConfiguration cfg){ -+ //NOP -+ } -+ -+ protected void configure(MemoryConfiguration mCfg){ -+ //NOP -+ } -+ -+ /** {@inheritDoc} */ -+ @Override protected void beforeTest() throws Exception { -+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false)); -+ -+ long seed = 1464583813940L; // System.currentTimeMillis(); -+ -+ info("Seed: " + seed + "L"); -+ -+ BPlusTree.rnd = new Random(seed); -+ -+ startGrids(gridCount()); -+ -+ awaitPartitionMapExchange(); -+ } -+ -+ /** {@inheritDoc} */ -+ @Override protected void afterTest() throws Exception { -+ BPlusTree.rnd = null; -+ -+ stopAllGrids(); -+ -+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false)); -+ } -+ -+ /** -+ * @return {@code True} if use large page. -+ */ -+ protected boolean isLargePage() { -+ return false; -+ } -+ -+ /** -+ * -+ */ -+ static class DbKey implements Serializable { -+ /** */ -+ int val; -+ -+ /** -+ * @param val Value. -+ */ -+ DbKey(int val) { -+ this.val = val; -+ } -+ -+ /** {@inheritDoc} */ -+ @Override public boolean equals(Object o) { -+ if (this == o) -+ return true; -+ -+ if (o == null || !(o instanceof DbKey)) -+ return false; -+ -+ DbKey key = (DbKey)o; -+ -+ return val == key.val; -+ } -+ -+ /** {@inheritDoc} */ -+ @Override public int hashCode() { -+ return val; -+ } -+ } -+ -+ /** -+ * -+ */ -+ static class LargeDbKey implements Serializable { -+ /** */ -+ int val; -+ -+ /** */ -+ byte[] data; -+ -+ /** -+ * @param val Value. -+ * @param size Key payload size. -+ */ -+ LargeDbKey(int val, int size) { -+ this.val = val; -+ -+ data = new byte[size]; -+ -+ Arrays.fill(data, (byte)val); -+ } -+ -+ /** {@inheritDoc} */ -+ @Override public boolean equals(Object o) { -+ if (this == o) -+ return true; -+ -+ if (o == null || !(o instanceof LargeDbKey)) -+ return false; -+ -+ LargeDbKey key = (LargeDbKey)o; -+ -+ return val == key.val && Arrays.equals(data, key.data); -+ } -+ -+ /** {@inheritDoc} */ -+ @Override public int hashCode() { -+ return val + Arrays.hashCode(data); -+ } -+ } -+ -+ /** -+ * -+ */ -+ static class DbValue implements Serializable { -+ /** */ -+ @QuerySqlField(index = true) -+ int iVal; -+ -+ /** */ -+ @QuerySqlField(index = true) -+ String sVal; -+ -+ /** */ -+ @QuerySqlField -+ long lVal; -+ -+ -+ -+ /** -+ * @param iVal Integer value. -+ * @param sVal String value. -+ * @param lVal Long value. -+ */ -+ DbValue(int iVal, String sVal, long lVal) { -+ this.iVal = iVal; -+ this.sVal = sVal; -+ this.lVal = lVal; -+ } -+ -+ /** {@inheritDoc} */ -+ @Override public boolean equals(Object o) { -+ if (this == o) -+ return true; -+ -+ if (o == null || getClass() != o.getClass()) -+ return false; -+ -+ DbValue dbVal = (DbValue)o; -+ -+ return iVal == dbVal.iVal && lVal == dbVal.lVal && -+ !(sVal != null ? !sVal.equals(dbVal.sVal) : dbVal.sVal != null); -+ } -+ -+ /** {@inheritDoc} */ -+ @Override public int hashCode() { -+ int res = iVal; -+ -+ res = 31 * res + (sVal != null ? sVal.hashCode() : 0); -+ res = 31 * res + (int)(lVal ^ (lVal >>> 32)); -+ -+ return res; -+ } -+ -+ /** {@inheritDoc} */ -+ @Override public String toString() { -+ return S.toString(DbValue.class, this); -+ } -+ } -+ -+ /** -+ * -+ */ -+ static class LargeDbValue { -+ /** */ -+ @QuerySqlField(index = true) -+ String str1; -+ -+ /** */ -+ @QuerySqlField(index = true) -+ String str2; -+ -+ /** */ -+ int[] arr; -+ -+ /** -+ * @param str1 String 1. -+ * @param str2 String 2. -+ * @param arr Big array. -+ */ -+ LargeDbValue(final String str1, final String str2, final int[] arr) { -+ this.str1 = str1; -+ this.str2 = str2; -+ this.arr = arr; -+ } -+ -+ /** {@inheritDoc} */ -+ @Override public boolean equals(final Object o) { -+ if (this == o) return true; -+ if (o == null || getClass() != o.getClass()) return false; -+ -+ final LargeDbValue that = (LargeDbValue) o; -+ -+ if (str1 != null ? !str1.equals(that.str1) : that.str1 != null) return false; -+ if (str2 != null ? !str2.equals(that.str2) : that.str2 != null) return false; -+ -+ return Arrays.equals(arr, that.arr); -+ -+ } -+ -+ /** {@inheritDoc} */ -+ @Override public int hashCode() { -+ int res = str1 != null ? str1.hashCode() : 0; -+ -+ res = 31 * res + (str2 != null ? str2.hashCode() : 0); -+ res = 31 * res + Arrays.hashCode(arr); -+ -+ return res; -+ } -+ -+ /** {@inheritDoc} */ -+ @Override public String toString() { -+ return S.toString(LargeDbValue.class, this); -+ } -+ } -+} -diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java -new file mode 100644 -index 0000000..6a5d039 ---- /dev/null -+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java -@@ -0,0 +1,84 @@ -+/* -+ * Licensed to the Apache Software Foundation (ASF) under one or more -+ * contributor license agreements. See the NOTICE file distributed with -+ * this work for additional information regarding copyright ownership. -+ * The ASF licenses this file to You under the Apache License, Version 2.0 -+ * (the "License"); you may not use this file except in compliance with -+ * the License. You may obtain a copy of the License at -+ * -+ * http://www.apache.org/licenses/LICENSE-2.0 -+ * -+ * Unless required by applicable law or agreed to in writing, software -+ * distributed under the License is distributed on an "AS IS" BASIS, -+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+ * See the License for the specific language governing permissions and -+ * limitations under the License. -+ */ -+ -+package org.apache.ignite.internal.processors.database; -+ -+import org.apache.ignite.Ignite; -+import org.apache.ignite.IgniteCompute; -+import org.apache.ignite.compute.ComputeTaskFuture; -+import org.apache.ignite.internal.IgniteEx; -+import org.apache.ignite.lang.IgniteRunnable; -+import org.apache.ignite.resources.IgniteInstanceResource; -+ -+import java.util.concurrent.TimeUnit; -+ -+/** -+ * -+ */ -+public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTest { -+ -+ /** Test duration in seconds*/ -+ protected abstract int duration(); -+ -+ @Override -+ protected long getTestTimeout() { -+ return duration() * 1200; -+ } -+ -+ /** */ -+ protected abstract void operation(IgniteEx ig); -+ -+ /** */ -+ public void testMemoryLeak() throws Exception { -+ -+ final long end = System.nanoTime() + TimeUnit.SECONDS.toNanos(duration()); -+ -+ int tasksCount = Runtime.getRuntime().availableProcessors() * 4; -+ -+ IgniteCompute compute = grid(0).compute().withAsync(); -+ -+ ComputeTaskFuture[] futs = new ComputeTaskFuture[tasksCount]; -+ -+ for (int i = 0; i < tasksCount; i++) { -+ compute.run(new IgniteRunnable() { -+ @IgniteInstanceResource -+ private Ignite ig; -+ -+ @Override -+ public void run() { -+ int i = 0; -+ while (System.nanoTime() < end) { -+ operation((IgniteEx) ig); -+ -+ if(i++ == 100) { -+ check((IgniteEx) ig); -+ i = 0; -+ } -+ } -+ } -+ }); -+ -+ futs[i] = compute.future(); -+ } -+ -+ for (ComputeTaskFuture fut : futs) { -+ fut.get(); -+ } -+ } -+ -+ protected void check(IgniteEx ig) {} -+} -diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java -new file mode 100644 -index 0000000..4cd74d0 ---- /dev/null -+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java -@@ -0,0 +1,85 @@ -+/* -+ * Licensed to the Apache Software Foundation (ASF) under one or more -+ * contributor license agreements. See the NOTICE file distributed with -+ * this work for additional information regarding copyright ownership. -+ * The ASF licenses this file to You under the Apache License, Version 2.0 -+ * (the "License"); you may not use this file except in compliance with -+ * the License. You may obtain a copy of the License at -+ * -+ * http://www.apache.org/licenses/LICENSE-2.0 -+ * -+ * Unless required by applicable law or agreed to in writing, software -+ * distributed under the License is distributed on an "AS IS" BASIS, -+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+ * See the License for the specific language governing permissions and -+ * limitations under the License. -+ */ -+ -+package org.apache.ignite.internal.processors.database; -+ -+import org.apache.ignite.IgniteCache; -+import org.apache.ignite.configuration.IgniteConfiguration; -+import org.apache.ignite.configuration.MemoryConfiguration; -+import org.apache.ignite.internal.IgniteEx; -+ -+import java.util.Random; -+import java.util.concurrent.ThreadLocalRandom; -+ -+/** -+ * -+ */ -+public class IgniteDbMemoryLeakIndexedTest extends IgniteDbMemoryLeakAbstractTest { -+ -+ @Override -+ protected int duration() { -+ return 300; -+ } -+ -+ @Override -+ protected int gridCount() { -+ return 1; -+ } -+ -+ @Override -+ protected void configure(IgniteConfiguration cfg) { -+ cfg.setMetricsLogFrequency(5000); -+ } -+ -+ @Override -+ protected void configure(MemoryConfiguration mCfg) { -+ mCfg.setPageCacheSize(1024 * 1024); -+ } -+ -+ @Override -+ protected boolean indexingEnabled() { -+ return true; -+ } -+ -+ protected void operation(IgniteEx ig){ -+ IgniteCache<Object, Object> cache = ig.cache("non-primitive"); -+ Random rnd = ThreadLocalRandom.current(); -+ -+ for (int i = 0; i < 1000; i++) { -+ DbKey key = new DbKey(rnd.nextInt(200_000)); -+ -+ DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500)); -+ -+ switch (rnd.nextInt(3)) { -+ case 0: -+ cache.getAndPut(key, v0); -+ case 1: -+ cache.get(key); -+ break; -+ case 2: -+ cache.getAndRemove(key); -+ } -+ } -+ } -+ -+ @Override -+ protected void check(IgniteEx ig) { -+ long pages = ig.context().cache().context().database().pageMemory().loadedPages(); -+ -+ assertTrue(pages < 19100); -+ } -+} -diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java -new file mode 100644 -index 0000000..a4d88e1 ---- /dev/null -+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java -@@ -0,0 +1,95 @@ -+/* -+ * Licensed to the Apache Software Foundation (ASF) under one or more -+ * contributor license agreements. See the NOTICE file distributed with -+ * this work for additional information regarding copyright ownership. -+ * The ASF licenses this file to You under the Apache License, Version 2.0 -+ * (the "License"); you may not use this file except in compliance with -+ * the License. You may obtain a copy of the License at -+ * -+ * http://www.apache.org/licenses/LICENSE-2.0 -+ * -+ * Unless required by applicable law or agreed to in writing, software -+ * distributed under the License is distributed on an "AS IS" BASIS, -+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+ * See the License for the specific language governing permissions and -+ * limitations under the License. -+ */ -+ -+package org.apache.ignite.internal.processors.database; -+ -+import org.apache.ignite.IgniteCache; -+import org.apache.ignite.configuration.IgniteConfiguration; -+import org.apache.ignite.configuration.MemoryConfiguration; -+import org.apache.ignite.internal.IgniteEx; -+ -+import java.util.Random; -+import java.util.concurrent.ThreadLocalRandom; -+ -+/** -+ * -+ */ -+public class IgniteDbMemoryLeakLargeObjectsTest extends IgniteDbMemoryLeakAbstractTest { -+ -+ private final static int[] ARRAY; -+ static { -+ ARRAY = new int[1024]; -+ Random rnd = new Random(); -+ for (int i = 0; i < ARRAY.length; i++) { -+ ARRAY[i] = rnd.nextInt(); -+ } -+ -+ } -+ -+ @Override -+ protected int duration() { -+ return 300; -+ } -+ -+ @Override -+ protected int gridCount() { -+ return 1; -+ } -+ -+ @Override -+ protected void configure(IgniteConfiguration cfg) { -+ cfg.setMetricsLogFrequency(5000); -+ } -+ -+ @Override -+ protected void configure(MemoryConfiguration mCfg) { -+ mCfg.setPageCacheSize(60 * 1024 * 1024); -+ } -+ -+ @Override -+ protected boolean indexingEnabled() { -+ return false; -+ } -+ -+ protected void operation(IgniteEx ig){ -+ IgniteCache<Object, Object> cache = ig.cache("large"); -+ Random rnd = ThreadLocalRandom.current(); -+ -+ for (int i = 0; i < 1000; i++) { -+ LargeDbKey key = new LargeDbKey(rnd.nextInt(10_000), 1024); -+ -+ LargeDbValue v0 = new LargeDbValue("test-value-1-" + rnd.nextInt(200), "test-value-2-" + rnd.nextInt(200), ARRAY); -+ -+ switch (rnd.nextInt(3)) { -+ case 0: -+ cache.getAndPut(key, v0); -+ case 1: -+ cache.get(key); -+ break; -+ case 2: -+ cache.getAndRemove(key); -+ } -+ } -+ } -+ -+ @Override -+ protected void check(IgniteEx ig) { -+ long pages = ig.context().cache().context().database().pageMemory().loadedPages(); -+ -+ assertTrue(pages < 50000); -+ } -+} -diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java -new file mode 100644 -index 0000000..bfa4aa9 ---- /dev/null -+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java -@@ -0,0 +1,90 @@ -+/* -+ * Licensed to the Apache Software Foundation (ASF) under one or more -+ * contributor license agreements. See the NOTICE file distributed with -+ * this work for additional information regarding copyright ownership. -+ * The ASF licenses this file to You under the Apache License, Version 2.0 -+ * (the "License"); you may not use this file except in compliance with -+ * the License. You may obtain a copy of the License at -+ * -+ * http://www.apache.org/licenses/LICENSE-2.0 -+ * -+ * Unless required by applicable law or agreed to in writing, software -+ * distributed under the License is distributed on an "AS IS" BASIS, -+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+ * See the License for the specific language governing permissions and -+ * limitations under the License. -+ */ -+ -+package org.apache.ignite.internal.processors.database; -+ -+import org.apache.ignite.IgniteCache; -+import org.apache.ignite.configuration.IgniteConfiguration; -+import org.apache.ignite.configuration.MemoryConfiguration; -+import org.apache.ignite.internal.IgniteEx; -+ -+import java.util.Random; -+import java.util.concurrent.ThreadLocalRandom; -+ -+/** -+ * -+ */ -+public class IgniteDbMemoryLeakLargePagesTest extends IgniteDbMemoryLeakAbstractTest { -+ -+ @Override -+ protected int duration() { -+ return 300; -+ } -+ -+ @Override -+ protected int gridCount() { -+ return 1; -+ } -+ -+ @Override -+ protected void configure(IgniteConfiguration cfg) { -+ cfg.setMetricsLogFrequency(5000); -+ } -+ -+ @Override -+ protected void configure(MemoryConfiguration mCfg) { -+ mCfg.setPageCacheSize(100 * 1024 * 1024); -+ } -+ -+ @Override -+ protected boolean indexingEnabled() { -+ return false; -+ } -+ -+ @Override -+ protected boolean isLargePage() { -+ return true; -+ } -+ -+ protected void operation(IgniteEx ig){ -+ IgniteCache<Object, Object> cache = ig.cache("non-primitive"); -+ Random rnd = ThreadLocalRandom.current(); -+ -+ for (int i = 0; i < 1000; i++) { -+ DbKey key = new DbKey(rnd.nextInt(200_000)); -+ -+ DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500)); -+ -+ switch (rnd.nextInt(3)) { -+ case 0: -+ cache.getAndPut(key, v0); -+ case 1: -+ cache.get(key); -+ break; -+ case 2: -+ cache.getAndRemove(key); -+ } -+ } -+ } -+ -+ @Override -+ protected void check(IgniteEx ig) { -+ long pages = ig.context().cache().context().database().pageMemory().loadedPages(); -+ -+ assertTrue(pages < 4600); -+ } -+} -diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java -new file mode 100644 -index 0000000..6af4e41 ---- /dev/null -+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java -@@ -0,0 +1,85 @@ -+/* -+ * Licensed to the Apache Software Foundation (ASF) under one or more -+ * contributor license agreements. See the NOTICE file distributed with -+ * this work for additional information regarding copyright ownership. -+ * The ASF licenses this file to You under the Apache License, Version 2.0 -+ * (the "License"); you may not use this file except in compliance with -+ * the License. You may obtain a copy of the License at -+ * -+ * http://www.apache.org/licenses/LICENSE-2.0 -+ * -+ * Unless required by applicable law or agreed to in writing, software -+ * distributed under the License is distributed on an "AS IS" BASIS, -+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+ * See the License for the specific language governing permissions and -+ * limitations under the License. -+ */ -+ -+package org.apache.ignite.internal.processors.database; -+ -+import org.apache.ignite.IgniteCache; -+import org.apache.ignite.configuration.IgniteConfiguration; -+import org.apache.ignite.configuration.MemoryConfiguration; -+import org.apache.ignite.internal.IgniteEx; -+ -+import java.util.Random; -+import java.util.concurrent.ThreadLocalRandom; -+ -+/** -+ * -+ */ -+public class IgniteDbMemoryLeakTest extends IgniteDbMemoryLeakAbstractTest { -+ -+ @Override -+ protected int duration() { -+ return 300; -+ } -+ -+ @Override -+ protected int gridCount() { -+ return 1; -+ } -+ -+ @Override -+ protected void configure(IgniteConfiguration cfg) { -+ cfg.setMetricsLogFrequency(5000); -+ } -+ -+ @Override -+ protected void configure(MemoryConfiguration mCfg) { -+ mCfg.setPageCacheSize(1024 * 1024); -+ } -+ -+ @Override -+ protected boolean indexingEnabled() { -+ return false; -+ } -+ -+ protected void operation(IgniteEx ig){ -+ IgniteCache<Object, Object> cache = ig.cache("non-primitive"); -+ Random rnd = ThreadLocalRandom.current(); -+ -+ for (int i = 0; i < 1000; i++) { -+ DbKey key = new DbKey(rnd.nextInt(200_000)); -+ -+ DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500)); -+ -+ switch (rnd.nextInt(3)) { -+ case 0: -+ cache.getAndPut(key, v0); -+ case 1: -+ cache.get(key); -+ break; -+ case 2: -+ cache.getAndRemove(key); -+ } -+ } -+ } -+ -+ @Override -+ protected void check(IgniteEx ig) { -+ long pages = ig.context().cache().context().database().pageMemory().loadedPages(); -+ -+ assertTrue(pages < 19100); -+ } -+} -diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java -new file mode 100644 -index 0000000..d9e3f34 ---- /dev/null -+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java -@@ -0,0 +1,92 @@ -+/* -+ * Licensed to the Apache Software Foundation (ASF) under one or more -+ * contributor license agreements. See the NOTICE file distributed with -+ * this work for additional information regarding copyright ownership. -+ * The ASF licenses this file to You under the Apache License, Version 2.0 -+ * (the "License"); you may not use this file except in compliance with -+ * the License. You may obtain a copy of the License at -+ * -+ * http://www.apache.org/licenses/LICENSE-2.0 -+ * -+ * Unless required by applicable law or agreed to in writing, software -+ * distributed under the License is distributed on an "AS IS" BASIS, -+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+ * See the License for the specific language governing permissions and -+ * limitations under the License. -+ */ -+ -+package org.apache.ignite.internal.processors.database; -+ -+import org.apache.ignite.IgniteCache; -+import org.apache.ignite.configuration.IgniteConfiguration; -+import org.apache.ignite.configuration.MemoryConfiguration; -+import org.apache.ignite.internal.IgniteEx; -+ -+import javax.cache.expiry.CreatedExpiryPolicy; -+import javax.cache.expiry.Duration; -+import javax.cache.expiry.ExpiryPolicy; -+import java.util.Random; -+import java.util.concurrent.ThreadLocalRandom; -+ -+import static java.util.concurrent.TimeUnit.MILLISECONDS; -+ -+/** -+ * -+ */ -+public class IgniteDbMemoryLeakWithExpirationTest extends IgniteDbMemoryLeakAbstractTest { -+ -+ private static final ExpiryPolicy EXPIRY = new CreatedExpiryPolicy(new Duration(MILLISECONDS, 10L)); -+ -+ @Override -+ protected int duration() { -+ return 300; -+ } -+ -+ @Override -+ protected int gridCount() { -+ return 1; -+ } -+ -+ @Override -+ protected void configure(IgniteConfiguration cfg) { -+ cfg.setMetricsLogFrequency(5000); -+ } -+ -+ @Override -+ protected void configure(MemoryConfiguration mCfg) { -+ mCfg.setPageCacheSize(1024 * 1024); -+ } -+ -+ @Override -+ protected boolean indexingEnabled() { -+ return false; -+ } -+ -+ protected void operation(IgniteEx ig) { -+ IgniteCache<Object, Object> cache = ig.cache("non-primitive").withExpiryPolicy(EXPIRY); -+ Random rnd = ThreadLocalRandom.current(); -+ -+ for (int i = 0; i < 1000; i++) { -+ DbKey key = new DbKey(rnd.nextInt(200_000)); -+ -+ DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500)); -+ -+ switch (rnd.nextInt(3)) { -+ case 0: -+ cache.getAndPut(key, v0); -+ case 1: -+ cache.get(key); -+ break; -+ case 2: -+ cache.getAndRemove(key); -+ } -+ } -+ } -+ -+ @Override -+ protected void check(IgniteEx ig) { -+ long pages = ig.context().cache().context().database().pageMemory().loadedPages(); -+ -+ assertTrue(pages < 10000); -+ } -+} -diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java -index c7a07e3..228a262 100644 ---- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java -+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java -@@ -17,175 +17,39 @@ - - package org.apache.ignite.internal.processors.database; - --import java.io.Serializable; --import java.util.Arrays; --import java.util.HashMap; --import java.util.HashSet; --import java.util.LinkedHashMap; --import java.util.List; --import java.util.Map; --import java.util.Random; --import java.util.Set; --import java.util.UUID; --import java.util.concurrent.ThreadLocalRandom; --import javax.cache.Cache; - import org.apache.ignite.Ignite; - import org.apache.ignite.IgniteCache; - import org.apache.ignite.IgniteDataStreamer; --import org.apache.ignite.cache.CacheAtomicityMode; - import org.apache.ignite.cache.CachePeekMode; --import org.apache.ignite.cache.CacheRebalanceMode; --import org.apache.ignite.cache.CacheWriteSynchronizationMode; - import org.apache.ignite.cache.affinity.Affinity; --import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; --import org.apache.ignite.cache.affinity.AffinityFunction; - import org.apache.ignite.cache.query.QueryCursor; - import org.apache.ignite.cache.query.ScanQuery; - import org.apache.ignite.cache.query.SqlFieldsQuery; - import org.apache.ignite.cache.query.SqlQuery; --import org.apache.ignite.cache.query.annotations.QuerySqlField; --import org.apache.ignite.configuration.CacheConfiguration; --import org.apache.ignite.configuration.MemoryConfiguration; --import org.apache.ignite.configuration.IgniteConfiguration; - import org.apache.ignite.internal.IgniteEx; - import org.apache.ignite.internal.processors.cache.GridCacheAdapter; - import org.apache.ignite.internal.processors.cache.database.tree.BPlusTree; - import org.apache.ignite.internal.util.GridRandom; - import org.apache.ignite.internal.util.typedef.PA; - import org.apache.ignite.internal.util.typedef.X; --import org.apache.ignite.internal.util.typedef.internal.S; --import org.apache.ignite.internal.util.typedef.internal.U; --import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; --import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; --import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; - import org.apache.ignite.testframework.GridTestUtils; --import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; - import org.junit.Assert; - -+import javax.cache.Cache; -+import java.util.HashMap; -+import java.util.HashSet; -+import java.util.LinkedHashMap; -+import java.util.List; -+import java.util.Map; -+import java.util.Random; -+import java.util.Set; -+import java.util.UUID; -+import java.util.concurrent.ThreadLocalRandom; -+ - /** - * - */ --public abstract class IgniteDbPutGetAbstractTest extends GridCommonAbstractTest { -- /** */ -- private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); -- -- /** -- * @return Node count. -- */ -- protected abstract int gridCount(); -- -- /** -- * @return {@code True} if indexing is enabled. -- */ -- protected abstract boolean indexingEnabled(); -- -- /** {@inheritDoc} */ -- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { -- IgniteConfiguration cfg = super.getConfiguration(gridName); -- -- MemoryConfiguration dbCfg = new MemoryConfiguration(); -- -- if (isLargePage()) { -- dbCfg.setConcurrencyLevel(Runtime.getRuntime().availableProcessors() * 4); -- -- dbCfg.setPageSize(16 * 1024); -- -- dbCfg.setPageCacheSize(200 * 1024 * 1024); -- } -- else { -- dbCfg.setConcurrencyLevel(Runtime.getRuntime().availableProcessors() * 4); -- -- dbCfg.setPageSize(1024); -- -- dbCfg.setPageCacheSize(200 * 1024 * 1024); -- } -- -- cfg.setMemoryConfiguration(dbCfg); -- -- CacheConfiguration ccfg = new CacheConfiguration(); -- -- if (indexingEnabled()) -- ccfg.setIndexedTypes(Integer.class, DbValue.class); -- -- ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); -- ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); -- ccfg.setRebalanceMode(CacheRebalanceMode.SYNC); -- ccfg.setAffinity(new RendezvousAffinityFunction(false, 32)); -- -- CacheConfiguration ccfg2 = new CacheConfiguration("non-primitive"); -- -- if (indexingEnabled()) -- ccfg2.setIndexedTypes(DbKey.class, DbValue.class); -- -- ccfg2.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); -- ccfg2.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); -- ccfg2.setRebalanceMode(CacheRebalanceMode.SYNC); -- ccfg2.setAffinity(new RendezvousAffinityFunction(false, 32)); -- -- CacheConfiguration ccfg3 = new CacheConfiguration("large"); -- -- if (indexingEnabled()) -- ccfg3.setIndexedTypes(Integer.class, LargeDbValue.class); -- -- ccfg3.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); -- ccfg3.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); -- ccfg3.setRebalanceMode(CacheRebalanceMode.SYNC); -- ccfg3.setAffinity(new RendezvousAffinityFunction(false, 32)); -- -- CacheConfiguration ccfg4 = new CacheConfiguration("tiny"); -- -- ccfg4.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); -- ccfg4.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); -- ccfg4.setRebalanceMode(CacheRebalanceMode.SYNC); -- ccfg4.setAffinity(new RendezvousAffinityFunction(false, 32)); -- -- final AffinityFunction aff = new RendezvousAffinityFunction(1, null); -- -- ccfg4.setAffinity(aff); -- -- cfg.setCacheConfiguration(ccfg, ccfg2, ccfg3, ccfg4); -- -- TcpDiscoverySpi discoSpi = new TcpDiscoverySpi(); -- -- discoSpi.setIpFinder(IP_FINDER); -- -- cfg.setDiscoverySpi(discoSpi); -- cfg.setMarshaller(null); -- -- return cfg; -- } -- -- /** {@inheritDoc} */ -- @Override protected void beforeTest() throws Exception { -- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false)); -- -- long seed = 1464583813940L; // System.currentTimeMillis(); -- -- info("Seed: " + seed + "L"); -- -- BPlusTree.rnd = new Random(seed); -- -- startGrids(gridCount()); -- -- awaitPartitionMapExchange(); -- } -- -- /** {@inheritDoc} */ -- @Override protected void afterTest() throws Exception { -- BPlusTree.rnd = null; -- -- stopAllGrids(); -- -- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false)); -- } -- -- /** -- * @return {@code True} if use large page. -- */ -- protected boolean isLargePage() { -- return false; -- }; -- -+public abstract class IgniteDbPutGetAbstractTest extends IgniteDbAbstractTest { - /** - * - */ -@@ -1349,191 +1213,4 @@ private void checkEmpty(final GridCacheAdapter internalCache, final Object key) - - assertNull(internalCache.peekEx(key)); - } -- -- /** -- * -- */ -- private static class DbKey implements Serializable { -- /** */ -- private int val; -- -- /** -- * @param val Value. -- */ -- private DbKey(int val) { -- this.val = val; -- } -- -- /** {@inheritDoc} */ -- @Override public boolean equals(Object o) { -- if (this == o) -- return true; -- -- if (o == null || !(o instanceof DbKey)) -- return false; -- -- DbKey key = (DbKey)o; -- -- return val == key.val; -- } -- -- /** {@inheritDoc} */ -- @Override public int hashCode() { -- return val; -- } -- } -- -- /** -- * -- */ -- private static class LargeDbKey implements Serializable { -- /** */ -- private int val; -- -- /** */ -- private byte[] data; -- -- /** -- * @param val Value. -- * @param size Key payload size. -- */ -- private LargeDbKey(int val, int size) { -- this.val = val; -- -- data = new byte[size]; -- -- Arrays.fill(data, (byte)val); -- } -- -- /** {@inheritDoc} */ -- @Override public boolean equals(Object o) { -- if (this == o) -- return true; -- -- if (o == null || !(o instanceof LargeDbKey)) -- return false; -- -- LargeDbKey key = (LargeDbKey)o; -- -- return val == key.val && Arrays.equals(data, key.data); -- } -- -- /** {@inheritDoc} */ -- @Override public int hashCode() { -- return val + Arrays.hashCode(data); -- } -- } -- -- /** -- * -- */ -- private static class DbValue implements Serializable { -- /** */ -- @QuerySqlField(index = true) -- private int iVal; -- -- /** */ -- @QuerySqlField(index = true) -- private String sVal; -- -- /** */ -- @QuerySqlField -- private long lVal; -- -- /** -- * @param iVal Integer value. -- * @param sVal String value. -- * @param lVal Long value. -- */ -- public DbValue(int iVal, String sVal, long lVal) { -- this.iVal = iVal; -- this.sVal = sVal; -- this.lVal = lVal; -- } -- -- /** {@inheritDoc} */ -- @Override public boolean equals(Object o) { -- if (this == o) -- return true; -- -- if (o == null || getClass() != o.getClass()) -- return false; -- -- DbValue dbVal = (DbValue)o; -- -- return iVal == dbVal.iVal && lVal == dbVal.lVal && -- !(sVal != null ? !sVal.equals(dbVal.sVal) : dbVal.sVal != null); -- } -- -- /** {@inheritDoc} */ -- @Override public int hashCode() { -- int res = iVal; -- -- res = 31 * res + (sVal != null ? sVal.hashCode() : 0); -- res = 31 * res + (int)(lVal ^ (lVal >>> 32)); -- -- return res; -- } -- -- /** {@inheritDoc} */ -- @Override public String toString() { -- return S.toString(DbValue.class, this); -- } -- } -- -- /** -- * -- */ -- private static class LargeDbValue { -- /** */ -- @QuerySqlField(index = true) -- private String str1; -- -- /** */ -- @QuerySqlField(index = true) -- private String str2; -- -- /** */ -- private int[] arr; -- -- /** -- * @param str1 String 1. -- * @param str2 String 2. -- * @param arr Big array. -- */ -- public LargeDbValue(final String str1, final String str2, final int[] arr) { -- this.str1 = str1; -- this.str2 = str2; -- this.arr = arr; -- } -- -- /** {@inheritDoc} */ -- @Override public boolean equals(final Object o) { -- if (this == o) return true; -- if (o == null || getClass() != o.getClass()) return false; -- -- final LargeDbValue that = (LargeDbValue) o; -- -- if (str1 != null ? !str1.equals(that.str1) : that.str1 != null) return false; -- if (str2 != null ? !str2.equals(that.str2) : that.str2 != null) return false; -- -- return Arrays.equals(arr, that.arr); -- -- } -- -- /** {@inheritDoc} */ -- @Override public int hashCode() { -- int res = str1 != null ? str1.hashCode() : 0; -- -- res = 31 * res + (str2 != null ? str2.hashCode() : 0); -- res = 31 * res + Arrays.hashCode(arr); -- -- return res; -- } -- -- /** {@inheritDoc} */ -- @Override public String toString() { -- return S.toString(LargeDbValue.class, this); -- } -- } - } - -From 8e12097f9094d7f155135ee2f4c9c33f5f7af9aa Mon Sep 17 00:00:00 2001 -From: sboikov <[email protected]> -Date: Wed, 15 Feb 2017 15:08:14 +0300 -Subject: [PATCH 02/41] ignite-4694 review - ---- - .../database/IgniteDbMemoryLeakAbstractTest.java | 15 ++++++++------- - .../database/IgniteDbMemoryLeakIndexedTest.java | 3 +++ - .../database/IgniteDbMemoryLeakLargeObjectsTest.java | 9 +++++---- - .../database/IgniteDbMemoryLeakLargePagesTest.java | 2 ++ - .../processors/database/IgniteDbMemoryLeakTest.java | 10 +++++++--- - .../database/IgniteDbMemoryLeakWithExpirationTest.java | 2 +- - 6 files changed, 26 insertions(+), 15 deletions(-) - -diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java -index 6a5d039..fc0e715 100644 ---- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java -+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java -@@ -27,10 +27,10 @@ - import java.util.concurrent.TimeUnit; - - /** -- * -+ * TODO: fix javadoc warnings, code style. - */ - public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTest { -- -+ // TODO: take duration from system property. - /** Test duration in seconds*/ - protected abstract int duration(); - -@@ -44,16 +44,18 @@ protected long getTestTimeout() { - - /** */ - public void testMemoryLeak() throws Exception { -+ // TODO: take PageMemory max size is the same as we configured. - - final long end = System.nanoTime() + TimeUnit.SECONDS.toNanos(duration()); - -- int tasksCount = Runtime.getRuntime().availableProcessors() * 4; -+ // TODO: use threads instead of compute or make sure there are enough threads in pool. -+ int tasksCnt = Runtime.getRuntime().availableProcessors() * 4; - - IgniteCompute compute = grid(0).compute().withAsync(); - -- ComputeTaskFuture[] futs = new ComputeTaskFuture[tasksCount]; -+ ComputeTaskFuture[] futs = new ComputeTaskFuture[tasksCnt]; - -- for (int i = 0; i < tasksCount; i++) { -+ for (int i = 0; i < tasksCnt; i++) { - compute.run(new IgniteRunnable() { - @IgniteInstanceResource - private Ignite ig; -@@ -75,9 +77,8 @@ public void run() { - futs[i] = compute.future(); - } - -- for (ComputeTaskFuture fut : futs) { -+ for (ComputeTaskFuture fut : futs) - fut.get(); -- } - } - - protected void check(IgniteEx ig) {} -diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java -index 4cd74d0..db77131 100644 ---- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java -+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java -@@ -51,6 +51,7 @@ protected void configure(MemoryConfiguration mCfg) { - } - - @Override -+ // TODO: move test to module ignite-indexing. - protected boolean indexingEnabled() { - return true; - } -@@ -64,6 +65,8 @@ protected void operation(IgniteEx ig){ - - DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500)); - -+ // TODO: also execute sql queries. -+ - switch (rnd.nextInt(3)) { - case 0: - cache.getAndPut(key, v0); -diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java -index a4d88e1..2a6c8cd 100644 ---- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java -+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java -@@ -29,15 +29,16 @@ - * - */ - public class IgniteDbMemoryLeakLargeObjectsTest extends IgniteDbMemoryLeakAbstractTest { -- -+ /** */ - private final static int[] ARRAY; -+ - static { - ARRAY = new int[1024]; -+ - Random rnd = new Random(); -- for (int i = 0; i < ARRAY.length; i++) { -- ARRAY[i] = rnd.nextInt(); -- } - -+ for (int i = 0; i < ARRAY.length; i++) -+ ARRAY[i] = rnd.nextInt(); - } - - @Override -diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java -index bfa4aa9..91c96af 100644 ---- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java -+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java -@@ -47,6 +47,7 @@ protected void configure(IgniteConfiguration cfg) { - - @Override - protected void configure(MemoryConfiguration mCfg) { -+ // TODO: understand why such overhead with large pages. - mCfg.setPageCacheSize(100 * 1024 * 1024); - } - -@@ -60,6 +61,7 @@ protected boolean isLargePage() { - return true; - } - -+ // TODO: avoid copy/paste. - protected void operation(IgniteEx ig){ - IgniteCache<Object, Object> cache = ig.cache("non-primitive"); - Random rnd = ThreadLocalRandom.current(); -diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java -index 6af4e41..2b0ce1e 100644 ---- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java -+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java -@@ -29,9 +29,8 @@ - * - */ - public class IgniteDbMemoryLeakTest extends IgniteDbMemoryLeakAbstractTest { -- -- @Override -- protected int duration() { -+ /** {@inheritDoc} */ -+ @Override protected int duration() { - return 300; - } - -@@ -64,12 +63,17 @@ protected void operation(IgniteEx ig){ - - DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500)); - -+ // TODO: also execute scan query. -+ - switch (rnd.nextInt(3)) { - case 0: - cache.getAndPut(key, v0); -+ break; -+ - case 1: - cache.get(key); - break; -+ - case 2: - cache.getAndRemove(key); - } -diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java -index d9e3f34..95fe8c8 100644 ---- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java -+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java -@@ -34,7 +34,7 @@ - * - */ - public class IgniteDbMemoryLeakWithExpirationTest extends IgniteDbMemoryLeakAbstractTest { -- -+ /** */ - private static final ExpiryPolicy EXPIRY = new CreatedExpiryPolicy(new Duration(MILLISECONDS, 10L)); - - @Override - -From e70d990f14288cfc8fe211fa25631016d5708144 Mon Sep 17 00:00:00 2001 -From: Igor Seliverstov <[email protected]> -Date: Wed, 15 Feb 2017 18:04:38 +0300 -Subject: [PATCH 03/41] IGNITE-4694 Add tests to check there are no memory - leaks in PageMemory - ---- - .../cache/IgniteCacheOffheapManagerImpl.java | 2 +- - .../processors/database/IgniteDbAbstractTest.java | 6 + - .../database/IgniteDbMemoryLeakAbstractTest.java | 172 ++++++++++++++++----- - .../database/IgniteDbMemoryLeakIndexedTest.java | 65 +------- - .../IgniteDbMemoryLeakLargeObjectsTest.java | 64 ++------ - .../database/IgniteDbMemoryLeakLargePagesTest.java | 67 ++------ - .../database/IgniteDbMemoryLeakTest.java | 63 ++------ - .../IgniteDbMemoryLeakWithExpirationTest.java | 58 +------ - 8 files changed, 182 insertions(+), 315 deletions(-) - -diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java -index 5df99b6..9becc99 100644 ---- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java -+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java -@@ -897,7 +897,7 @@ public CacheDataStoreImpl( - */ - private boolean canUpdateOldRow(@Nullable CacheDataRow oldRow, DataRow dataRow) - throws IgniteCheckedException { -- if (oldRow == null || indexingEnabled) -+ if (oldRow == null || indexingEnabled || oldRow.expireTime() != dataRow.expireTime()) - return false; - - CacheObjectContext coCtx = cctx.cacheObjectContext(); -diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java -index 3bc7004..9297cec 100644 ---- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java -+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java -@@ -130,10 +130,16 @@ - return cfg; - } - -+ /** -+ * @param cfg IgniteConfiguration. -+ */ - protected void configure(IgniteConfiguration cfg){ - //NOP - } - -+ /** -+ * @param mCfg MemoryConfiguration. -+ */ - protected void configure(MemoryConfiguration mCfg){ - //NOP - } -diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java -index fc0e715..bca3af0 100644 ---- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java -+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java -@@ -17,69 +17,157 @@ - - package org.apache.ignite.internal.processors.database; - --import org.apache.ignite.Ignite; --import org.apache.ignite.IgniteCompute; --import org.apache.ignite.compute.ComputeTaskFuture; --import org.apache.ignite.internal.IgniteEx; --import org.apache.ignite.lang.IgniteRunnable; --import org.apache.ignite.resources.IgniteInstanceResource; -- -+import java.util.Random; - import java.util.concurrent.TimeUnit; -+import org.apache.ignite.IgniteCache; -+import org.apache.ignite.configuration.IgniteConfiguration; -+import org.apache.ignite.configuration.MemoryConfiguration; -+import org.apache.ignite.internal.IgniteEx; -+import org.apache.ignite.internal.util.GridRandom; -+import org.jetbrains.annotations.NotNull; - - /** -- * TODO: fix javadoc warnings, code style. -+ * Base class for memory leaks tests. - */ - public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTest { -- // TODO: take duration from system property. -- /** Test duration in seconds*/ -- protected abstract int duration(); -- -- @Override -- protected long getTestTimeout() { -- return duration() * 1200; -- } - - /** */ -- protected abstract void operation(IgniteEx ig); -+ private volatile Exception ex = null; - - /** */ -- public void testMemoryLeak() throws Exception { -- // TODO: take PageMemory max size is the same as we configured. -+ private static final ThreadLocal<Random> THREAD_LOCAL_RANDOM = new ThreadLocal<>(); - -- final long end = System.nanoTime() + TimeUnit.SECONDS.toNanos(duration()); -+ /** {@inheritDoc} */ -+ @Override protected void configure(IgniteConfiguration cfg) { -+ cfg.setMetricsLogFrequency(5000); -+ } - -- // TODO: use threads instead of compute or make sure there are enough threads in pool. -- int tasksCnt = Runtime.getRuntime().availableProcessors() * 4; -+ /** {@inheritDoc} */ -+ @Override protected void configure(MemoryConfiguration mCfg) { -+ int concLvl = Runtime.getRuntime().availableProcessors(); - -- IgniteCompute compute = grid(0).compute().withAsync(); -+ mCfg.setConcurrencyLevel(concLvl); -+ mCfg.setPageCacheSize(1024 * 1024 * concLvl); //minimal possible value -+ } - -- ComputeTaskFuture[] futs = new ComputeTaskFuture[tasksCnt]; -+ /** -+ * @return Test duration in seconds. -+ */ -+ protected int duration() { -+ return 300; -+ } -+ -+ /** {@inheritDoc} */ -+ @Override protected int gridCount() { -+ return 1; -+ } -+ -+ /** {@inheritDoc} */ -+ @Override protected boolean indexingEnabled() { -+ return false; -+ } - -- for (int i = 0; i < tasksCnt; i++) { -- compute.run(new IgniteRunnable() { -- @IgniteInstanceResource -- private Ignite ig; -+ /** {@inheritDoc} */ -+ @Override protected long getTestTimeout() { -+ return (duration() + 1) * 1000; -+ } - -- @Override -- public void run() { -- int i = 0; -- while (System.nanoTime() < end) { -- operation((IgniteEx) ig); -+ /** -+ * @param ig Ignite instance. -+ * @return IgniteCache. -+ */ -+ protected abstract IgniteCache<Object,Object> cache(IgniteEx ig); -+ -+ /** -+ * @return Cache key to perform an operation. -+ */ -+ protected abstract Object key(); -+ -+ /** -+ * @return Cache value to perform an operation. -+ * @param key Cache key to perform an operation. -+ */ -+ protected abstract Object value(Object key); -+ -+ /** -+ * @param cache IgniteCache. -+ */ -+ protected void operation(IgniteCache<Object, Object> cache) { -+ Object key = key(); -+ Object value = value(key); -+ -+ switch (getRandom().nextInt(3)) { -+ case 0: -+ cache.getAndPut(key, value); -+ case 1: -+ cache.get(key); -+ break; -+ case 2: -+ cache.getAndRemove(key); -+ } -+ } - -- if(i++ == 100) { -- check((IgniteEx) ig); -- i = 0; -- } -+ /** -+ * @return Random. -+ */ -+ @NotNull protected static Random getRandom() { -+ Random rnd = THREAD_LOCAL_RANDOM.get(); -+ -+ if(rnd == null){ -+ rnd = new GridRandom(); -+ THREAD_LOCAL_RANDOM.set(rnd); -+ } -+ -+ return rnd; -+ } -+ -+ /** -+ * @throws Exception If failed. -+ */ -+ public void testMemoryLeak() throws Exception { -+ final long end = System.nanoTime() + TimeUnit.SECONDS.toNanos(duration()); -+ -+ final IgniteEx ignite = grid(0); -+ final IgniteCache<Object, Object> cache = cache(ignite); -+ -+ Runnable target = new Runnable() { -+ @Override public void run() { -+ while (ex == null && System.nanoTime() < end) { -+ try { -+ operation(cache); -+ check(ignite); -+ } -+ catch (Exception e) { -+ ex = e; -+ -+ break; - } - } -- }); -+ } -+ }; -+ -+ Thread[] threads = new Thread[Runtime.getRuntime().availableProcessors()]; - -- futs[i] = compute.future(); -+ for (int i = 0; i < threads.length; i++) { -+ threads[i] = new Thread(target); -+ threads[i].start(); - } - -- for (ComputeTaskFuture fut : futs) -- fut.get(); -+ for (Thread thread : threads) { -+ thread.join(); -+ } -+ -+ if(ex != null){ -+ throw ex; -+ } - } - -- protected void check(IgniteEx ig) {} -+ /** -+ * Callback to check the current state -+ * -+ * @param ig Ignite instance -+ * @throws Exception If failed. -+ */ -+ protected void check(IgniteEx ig) throws Exception { -+ } - } -diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java -index db77131..acc6c2f 100644 ---- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java -+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java -@@ -17,72 +17,13 @@ - - package org.apache.ignite.internal.processors.database; - --import org.apache.ignite.IgniteCache; --import org.apache.ignite.configuration.IgniteConfiguration; --import org.apache.ignite.configuration.MemoryConfiguration; --import org.apache.ignite.internal.IgniteEx; -- --import java.util.Random; --import java.util.concurrent.ThreadLocalRandom; -- - /** - * - */ --public class IgniteDbMemoryLeakIndexedTest extends IgniteDbMemoryLeakAbstractTest { -- -- @Override -- protected int duration() { -- return 300; -- } -- -- @Override -- protected int gridCount() { -- return 1; -- } -- -- @Override -- protected void configure(IgniteConfiguration cfg) { -- cfg.setMetricsLogFrequency(5000); -- } -- -- @Override -- protected void configure(MemoryConfiguration mCfg) { -- mCfg.setPageCacheSize(1024 * 1024); -- } -+public class IgniteDbMemoryLeakIndexedTest extends IgniteDbMemoryLeakTest { - -- @Override -- // TODO: move test to module ignite-indexing. -- protected boolean indexingEnabled() { -+ /** {@inheritDoc} */ -+ @Override protected boolean indexingEnabled() { - return true; - } -- -- protected void operation(IgniteEx ig){ -- IgniteCache<Object, Object> cache = ig.cache("non-primitive"); -- Random rnd = ThreadLocalRandom.current(); -- -- for (int i = 0; i < 1000; i++) { -- DbKey key = new DbKey(rnd.nextInt(200_000)); -- -- DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500)); -- -- // TODO: also execute sql queries. -- -- switch (rnd.nextInt(3)) { -- case 0: -- cache.getAndPut(key, v0); -- case 1: -- cache.get(key); -- break; -- case 2: -- cache.getAndRemove(key); -- } -- } -- } -- -- @Override -- protected void check(IgniteEx ig) { -- long pages = ig.context().cache().context().database().pageMemory().loadedPages(); -- -- assertTrue(pages < 19100); -- } - } -diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java -index 2a6c8cd..8943743 100644 ---- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java -+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java -@@ -18,13 +18,8 @@ - package org.apache.ignite.internal.processors.database; - - import org.apache.ignite.IgniteCache; --import org.apache.ignite.configuration.IgniteConfiguration; --import org.apache.ignite.configuration.MemoryConfiguration; - import org.apache.ignite.internal.IgniteEx; - --import java.util.Random; --import java.util.concurrent.ThreadLocalRandom; -- - /** - * - */ -@@ -35,62 +30,29 @@ - static { - ARRAY = new int[1024]; - -- Random rnd = new Random(); -- - for (int i = 0; i < ARRAY.length; i++) -- ARRAY[i] = rnd.nextInt(); -- } -- -- @Override -- protected int duration() { -- return 300; -- } -- -- @Override -- protected int gridCount() { -- return 1; -+ ARRAY[i] = getRandom().nextInt(); - } - -- @Override -- protected void configure(IgniteConfiguration cfg) { -- cfg.setMetricsLogFrequency(5000); -+ /** {@inheritDoc} */ -+ @Override protected IgniteCache<Object, Object> cache(IgniteEx ig) { -+ return ig.cache("non-primitive"); - } - -- @Override -- protected void configure(MemoryConfiguration mCfg) { -- mCfg.setPageCacheSize(60 * 1024 * 1024); -+ /** {@inheritDoc} */ -+ @Override protected Object key() { -+ return new DbKey(getRandom().nextInt(200_000)); - } - -- @Override -- protected boolean indexingEnabled() { -- return false; -- } -- -- protected void operation(IgniteEx ig){ -- IgniteCache<Object, Object> cache = ig.cache("large"); -- Random rnd = ThreadLocalRandom.current(); -- -- for (int i = 0; i < 1000; i++) { -- LargeDbKey key = new LargeDbKey(rnd.nextInt(10_000), 1024); -- -- LargeDbValue v0 = new LargeDbValue("test-value-1-" + rnd.nextInt(200), "test-value-2-" + rnd.nextInt(200), ARRAY); -- -- switch (rnd.nextInt(3)) { -- case 0: -- cache.getAndPut(key, v0); -- case 1: -- cache.get(key); -- break; -- case 2: -- cache.getAndRemove(key); -- } -- } -+ /** {@inheritDoc} */ -+ @Override protected Object value(Object key) { -+ return new DbValue(((DbKey)key).val, "test-value-" + getRandom().nextInt(200), getRandom().nextInt(500)); - } - -- @Override -- protected void check(IgniteEx ig) { -+ /** {@inheritDoc} */ -+ @Override protected void check(IgniteEx ig) { - long pages = ig.context().cache().context().database().pageMemory().loadedPages(); - -- assertTrue(pages < 50000); -+ assertTrue(pages < 20000); - } - } -diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java -index 91c96af..8e4d0b4 100644 ---- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java -+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java -@@ -17,76 +17,31 @@ - - package org.apache.ignite.internal.processors.database; - --import org.apache.ignite.IgniteCache; --import org.apache.ignite.configuration.IgniteConfiguration; - import org.apache.ignite.configuration.MemoryConfiguration; - import org.apache.ignite.internal.IgniteEx; - --import java.util.Random; --import java.util.concurrent.ThreadLocalRandom; -- - /** - * - */ --public class IgniteDbMemoryLeakLargePagesTest extends IgniteDbMemoryLeakAbstractTest { -- -- @Override -- protected int duration() { -- return 300; -- } -+public class IgniteDbMemoryLeakLargePagesTest extends IgniteDbMemoryLeakTest { - -- @Override -- protected int gridCount() { -- return 1; -- } -+ /** {@inheritDoc} */ -+ @Override protected void configure(MemoryConfiguration mCfg) { -+ int concLvl = Runtime.getRuntime().availableProcessors(); -+ mCfg.setConcurrencyLevel(concLvl); -+ mCfg.setPageCacheSize(1024 * 1024 * concLvl * 16); - -- @Override -- protected void configure(IgniteConfiguration cfg) { -- cfg.setMetricsLogFrequency(5000); - } - -- @Override -- protected void configure(MemoryConfiguration mCfg) { -- // TODO: understand why such overhead with large pages. -- mCfg.setPageCacheSize(100 * 1024 * 1024); -- } -- -- @Override -- protected boolean indexingEnabled() { -- return false; -- } -- -- @Override -- protected boolean isLargePage() { -+ /** {@inheritDoc} */ -+ @Override protected boolean isLargePage() { - return true; - } - -- // TODO: avoid copy/paste. -- protected void operation(IgniteEx ig){ -- IgniteCache<Object, Object> cache = ig.cache("non-primitive"); -- Random rnd = ThreadLocalRandom.current(); -- -- for (int i = 0; i < 1000; i++) { -- DbKey key = new DbKey(rnd.nextInt(200_000)); -- -- DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500)); -- -- switch (rnd.nextInt(3)) { -- case 0: -- cache.getAndPut(key, v0); -- case 1: -- cache.get(key); -- break; -- case 2: -- cache.getAndRemove(key); -- } -- } -- } -- -- @Override -- protected void check(IgniteEx ig) { -+ /** {@inheritDoc} */ -+ @Override protected void check(IgniteEx ig) { - long pages = ig.context().cache().context().database().pageMemory().loadedPages(); - -- assertTrue(pages < 4600); -+ assertTrue(pages < 4000); - } - } -diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java -index 2b0ce1e..81d831b 100644 ---- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java -+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java -@@ -18,72 +18,31 @@ - package org.apache.ignite.internal.processors.database; - - import org.apache.ignite.IgniteCache; --import org.apache.ignite.configuration.IgniteConfiguration; --import org.apache.ignite.configuration.MemoryConfiguration; - import org.apache.ignite.internal.IgniteEx; - --import java.util.Random; --import java.util.concurrent.ThreadLocalRandom; -- - /** - * - */ - public class IgniteDbMemoryLeakTest extends IgniteDbMemoryLeakAbstractTest { - /** {@inheritDoc} */ -- @Override protected int duration() { -- return 300; -- } -- -- @Override -- protected int gridCount() { -- return 1; -- } -- -- @Override -- protected void configure(IgniteConfiguration cfg) { -- cfg.setMetricsLogFrequency(5000); -- } -- -- @Override -- protected void configure(MemoryConfiguration mCfg) { -- mCfg.setPageCacheSize(1024 * 1024); -+ @Override protected IgniteCache<Object, Object> cache(IgniteEx ig) { -+ return ig.cache("non-primitive"); - } - -- @Override -- protected boolean indexingEnabled() { -- return false; -+ /** {@inheritDoc} */ -+ @Override protected Object key() { -+ return new DbKey(getRandom().nextInt(200_000)); - } - -- protected void operation(IgniteEx ig){ -- IgniteCache<Object, Object> cache = ig.cache("non-primitive"); -- Random rnd = ThreadLocalRandom.current(); -- -- for (int i = 0; i < 1000; i++) { -- DbKey key = new DbKey(rnd.nextInt(200_000)); -- -- DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500)); -- -- // TODO: also execute scan query. -- -- switch (rnd.nextInt(3)) { -- case 0: -- cache.getAndPut(key, v0); -- break; -- -- case 1: -- cache.get(key); -- break; -- -- case 2: -- cache.getAndRemove(key); -- } -- } -+ /** {@inheritDoc} */ -+ @Override protected Object value(Object key) { -+ return new DbValue(((DbKey)key).val, "test-value-" + getRandom().nextInt(200), getRandom().nextInt(500)); - } - -- @Override -- protected void check(IgniteEx ig) { -+ /** {@inheritDoc} */ -+ @Override protected void check(IgniteEx ig) { - long pages = ig.context().cache().context().database().pageMemory().loadedPages(); - -- assertTrue(pages < 19100); -+ assertTrue(pages < 20000); - } - } -diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java -index 95fe8c8..a31ffb4 100644 ---- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java -+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java -@@ -18,75 +18,31 @@ - package org.apache.ignite.internal.processors.database; - - import org.apache.ignite.IgniteCache; --import org.apache.ignite.configuration.IgniteConfiguration; - import org.apache.ignite.configuration.MemoryConfiguration; - import org.apache.ignite.internal.IgniteEx; - - import javax.cache.expiry.CreatedExpiryPolicy; - import javax.cache.expiry.Duration; - import javax.cache.expiry.ExpiryPolicy; --import java.util.Random; --import java.util.concurrent.ThreadLocalRandom; - - import static java.util.concurrent.TimeUnit.MILLISECONDS; - - /** - * - */ --public class IgniteDbMemoryLeakWithExpirationTest extends IgniteDbMemoryLeakAbstractTest { -+public class IgniteDbMemoryLeakWithExpirationTest extends IgniteDbMemoryLeakTest { - /** */ - private static final ExpiryPolicy EXPIRY = new CreatedExpiryPolicy(new Duration(MILLISECONDS, 10L)); - -- @Override -- protected int duration() { -- return 300; -+ /** {@inheritDoc} */ -+ @Override protected IgniteCache<Object, Object> cache(IgniteEx ig) { -+ return ig.cache("non-primitive").withExpiryPolicy(EXPIRY); - } - -- @Override -- protected int gridCount() { -- return 1; -- } -- -- @Override -- protected void configure(IgniteConfiguration cfg) { -- cfg.setMetricsLogFrequency(5000); -- } -- -- @Override -- protected void configure(MemoryConfiguration mCfg) { -- mCfg.setPageCacheSize(1024 * 1024); -- } -- -- @Override -- protected boolean indexingEnabled() { -- return false; -- } -- -- protected void operation(IgniteEx ig) { -- IgniteCache<Object, Object> cache = ig.cache("non-primitive").withExpiryPolicy(EXPIRY); -- Random rnd = ThreadLocalRandom.current(); -- -- for (int i = 0; i < 1000; i++) { -- DbKey key = new DbKey(rnd.nextInt(200_000)); -- -- DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500)); -- -- switch (rnd.nextInt(3)) { -- case 0: -- cache.getAndPut(key, v0); -- case 1: -- cache.get(key); -- break; -- case 2: -- cache.getAndRemove(key); -- } -- } -- } -- -- @Override -- protected void check(IgniteEx ig) { -+ /** {@inheritDoc} */ -+ @Override protected void check(IgniteEx ig) { - long pages = ig.context().cache().context().database().pageMemory().loadedPages(); - -- assertTrue(pages < 10000); -+ assertTrue(pages < 7000); - } - } - -From 84c03e0c522abc90b2d91e514138eac08388abd2 Mon Sep 17 00:00:00 2001 -From: Igor Seliverstov <[email protected]> -Date: Thu, 16 Feb 2017 13:41:51 +0300 -Subject: [PATCH 04/41] IGNITE-4694 Add tests to check there are no memory - leaks in PageMemory (pending) - ---- - .../database/IgniteDbMemoryLeakAbstractTest.java | 118 +++++++++++++++++---- - .../database/IgniteDbMemoryLeakIndexedTest.java | 29 ----- - .../IgniteDbMemoryLeakLargeObjectsTest.java | 16 +-- - .../database/IgniteDbMemoryLeakLargePagesTest.java | 13 +-- - .../database/IgniteDbMemoryLeakTest.java | 14 ++- - .../IgniteDbMemoryLeakWithExpirationTest.java | 7 +- - .../database/IgniteDbMemoryLeakIndexedTest.java | 42 ++++++++ - 7 files changed, 165 insertions(+), 74 deletions(-) - delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java - create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java - -diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java -index bca3af0..819405e 100644 ---- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java -+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java -@@ -30,13 +30,37 @@ - * Base class for memory leaks tests. - */ - public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTest { -- -+ /** */ -+ @SuppressWarnings("WeakerAccess") protected static final int CONCURRENCY_LEVEL = 8; - /** */ - private volatile Exception ex = null; - - /** */ -+ private long warmUpEndTime; -+ -+ /** */ -+ private long endTime; -+ -+ /** */ -+ private long loadedPages = 0; -+ -+ /** */ -+ private long delta = 0; -+ -+ /** */ -+ private long probeCnt = 0; -+ -+ /** */ - private static final ThreadLocal<Random> THREAD_LOCAL_RANDOM = new ThreadLocal<>(); - -+ @Override protected void beforeTest() throws Exception { -+ super.beforeTest(); -+ -+ long startTime = System.nanoTime(); -+ warmUpEndTime = startTime + TimeUnit.SECONDS.toNanos(warmUp()); -+ endTime = warmUpEndTime + TimeUnit.SECONDS.toNanos(duration()); -+ } -+ - /** {@inheritDoc} */ - @Override protected void configure(IgniteConfiguration cfg) { - cfg.setMetricsLogFrequency(5000); -@@ -44,10 +68,7 @@ - - /** {@inheritDoc} */ - @Override protected void configure(MemoryConfiguration mCfg) { -- int concLvl = Runtime.getRuntime().availableProcessors(); -- -- mCfg.setConcurrencyLevel(concLvl); -- mCfg.setPageCacheSize(1024 * 1024 * concLvl); //minimal possible value -+ mCfg.setConcurrencyLevel(CONCURRENCY_LEVEL); - } - - /** -@@ -57,6 +78,13 @@ protected int duration() { - return 300; - } - -+ /** -+ * @return Warm up duration. -+ */ -+ @SuppressWarnings("WeakerAccess") protected int warmUp() { -+ return 300; -+ } -+ - /** {@inheritDoc} */ - @Override protected int gridCount() { - return 1; -@@ -69,14 +97,14 @@ protected int duration() { - - /** {@inheritDoc} */ - @Override protected long getTestTimeout() { -- return (duration() + 1) * 1000; -+ return (warmUp() + duration() + 1) * 1000; // One extra second to stop all threads - } - - /** - * @param ig Ignite instance. - * @return IgniteCache. - */ -- protected abstract IgniteCache<Object,Object> cache(IgniteEx ig); -+ protected abstract IgniteCache<Object, Object> cache(IgniteEx ig); - - /** - * @return Cache key to perform an operation. -@@ -84,8 +112,8 @@ protected int duration() { - protected abstract Object key(); - - /** -- * @return Cache value to perform an operation. - * @param key Cache key to perform an operation. -+ * @return Cache value to perform an operation. - */ - protected abstract Object value(Object key); - -@@ -99,8 +127,11 @@ protected void operation(IgniteCache<Object, Object> cache) { - switch (getRandom().nextInt(3)) { - case 0: - cache.getAndPut(key, value); -+ -+ break; - case 1: - cache.get(key); -+ - break; - case 2: - cache.getAndRemove(key); -@@ -113,7 +144,7 @@ protected void operation(IgniteCache<Object, Object> cache) { - @NotNull protected static Random getRandom() { - Random rnd = THREAD_LOCAL_RANDOM.get(); - -- if(rnd == null){ -+ if (rnd == null) { - rnd = new GridRandom(); - THREAD_LOCAL_RANDOM.set(rnd); - } -@@ -125,49 +156,96 @@ protected void operation(IgniteCache<Object, Object> cache) { - * @throws Exception If failed. - */ - public void testMemoryLeak() throws Exception { -- final long end = System.nanoTime() + TimeUnit.SECONDS.toNanos(duration()); - - final IgniteEx ignite = grid(0); - final IgniteCache<Object, Object> cache = cache(ignite); - - Runnable target = new Runnable() { - @Override public void run() { -- while (ex == null && System.nanoTime() < end) { -+ while (ex == null && System.nanoTime() < endTime) { - try { - operation(cache); -- check(ignite); - } - catch (Exception e) { - ex = e; -- - break; - } - } - } - }; - -- Thread[] threads = new Thread[Runtime.getRuntime().availableProcessors()]; -+ Thread[] threads = new Thread[CONCURRENCY_LEVEL]; -+ -+ info("Warming up is started."); - - for (int i = 0; i < threads.length; i++) { - threads[i] = new Thread(target); - threads[i].start(); - } - -- for (Thread thread : threads) { -- thread.join(); -+ Thread.sleep(TimeUnit.NANOSECONDS.toMillis(warmUpEndTime - System.nanoTime())); -+ -+ info("Warming up is ended."); -+ -+ while (System.nanoTime() < endTime) { -+ try { -+ check(ignite); -+ } -+ catch (Exception e) { -+ ex = e; -+ -+ break; -+ } -+ -+ Thread.sleep(TimeUnit.SECONDS.toMillis(5)); - } - -- if(ex != null){ -+ for (Thread thread : threads) -+ thread.join(); -+ -+ if (ex != null) - throw ex; -- } - } - - /** -- * Callback to check the current state -+ * Callback to check the current state. - * -- * @param ig Ignite instance -+ * @param ig Ignite instance. - * @throws Exception If failed. - */ - protected void check(IgniteEx ig) throws Exception { -+ long pagesActual = ig.context().cache().context().database().pageMemory().loadedPages(); -+ long pagesMax = pagesMax(); -+ -+ assertTrue( -+ "Maximal allowed pages number is exceeded. [allowed=" + pagesMax + "; actual= " + pagesActual + "]", -+ pagesActual <= pagesMax); -+ -+ if (loadedPages > 0) { -+ delta += pagesActual - loadedPages; -+ int allowedDelta = pagesDelta(); -+ -+ if(probeCnt++ > 12) { // we need some statistic first. Minimal statistic is taken for a minute. -+ long actualDelta = delta / probeCnt; -+ -+ assertTrue( -+ "Average growth pages in the number is more than expected. [allowed=" + allowedDelta + "; actual=" + actualDelta + "]", -+ actualDelta <= allowedDelta); -+ } -+ } -+ -+ loadedPages = pagesActual; -+ } -+ -+ /** -+ * @return Maximal allowed pages number. -+ */ -+ protected abstract long pagesMax(); -+ -+ /** -+ * @return Expected average number of pages, on which their total number can grow per 5 seconds. -+ */ -+ @SuppressWarnings("WeakerAccess") protected int pagesDelta() { -+ return 5; - } - } -diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java -deleted file mode 100644 -index acc6c2f..0000000 ---- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java -+++ /dev/null -@@ -1,29 +0,0 @@ --/* -- * Licensed to the Apache Software Foundation (ASF) under one or more -- * contributor license agreements. See the NOTICE file distributed with -- * this work for additional information regarding copyright ownership. -- * The ASF licenses this file to You under the Apache License, Version 2.0 -- * (the "License"); you may not use this file except in compliance with -- * the License. You may obtain a copy of the License at -- * -- * http://www.apache.org/licenses/LICENSE-2.0 -- * -- * Unless required by applicable law or agreed to in writing, software -- * distributed under the License is distributed on an "AS IS" BASIS, -- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -- * See the License for the specific language governing permissions and -- * limitations under the License. -- */ -- --package org.apache.ignite.internal.processors.database; -- --/** -- * -- */ --public class IgniteDbMemoryLeakIndexedTest extends IgniteDbMemoryLeakTest { -- -- /** {@inheritDoc} */ -- @Override protected boolean indexingEnabled() { -- return true; -- } --} -diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java -index 8943743..c1b1c9b 100644 ---- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java -+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java -@@ -36,23 +36,25 @@ - - /** {@inheritDoc} */ - @Override protected IgniteCache<Object, Object> cache(IgniteEx ig) { -- return ig.cache("non-primitive"); -+ return ig.cache("large"); - } - - /** {@inheritDoc} */ - @Override protected Object key() { -- return new DbKey(getRandom().nextInt(200_000)); -+ return new LargeDbKey(getRandom().nextInt(10_000), 1024); - } - - /** {@inheritDoc} */ - @Override protected Object value(Object key) { -- return new DbValue(((DbKey)key).val, "test-value-" + getRandom().nextInt(200), getRandom().nextInt(500)); -+ return new LargeDbValue("test-value-1-" + getRandom().nextInt(200), "test-value-2-" + getRandom().nextInt(200), ARRAY); -
<TRUNCATED>
