IGNITE-8069 IgniteOutOfMemoryException should be handled accordingly to 
provided failure handler

Signed-off-by: Andrey Gura <ag...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/9bb4ce8a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/9bb4ce8a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/9bb4ce8a

Branch: refs/heads/ignite-6083
Commit: 9bb4ce8ab9770967d39f0acac5cdc1dc4230abb4
Parents: c807ae9
Author: Aleksey Plekhanov <plehanov.a...@gmail.com>
Authored: Tue Apr 10 18:54:03 2018 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Tue Apr 10 18:54:03 2018 +0300

----------------------------------------------------------------------
 .../pagemem/impl/PageMemoryNoStoreImpl.java     |  17 ++-
 .../persistence/pagemem/PageMemoryImpl.java     |  17 ++-
 .../failure/AbstractFailureHandlerTest.java     |  74 ++++++++++
 .../ignite/failure/IoomFailureHandlerTest.java  | 144 +++++++++++++++++++
 .../persistence/pagemem/PageMemoryImplTest.java |   9 ++
 .../ignite/testsuites/IgniteBasicTestSuite.java |   2 +
 6 files changed, 259 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/9bb4ce8a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
----------------------------------------------------------------------
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
 
b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
index 7424af6..d4b22a6 100644
--- 
a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
+++ 
b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
@@ -28,6 +28,8 @@ import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
 import org.apache.ignite.internal.mem.DirectMemoryProvider;
 import org.apache.ignite.internal.mem.DirectMemoryRegion;
 import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
@@ -158,6 +160,9 @@ public class PageMemoryNoStoreImpl implements PageMemory {
     /** */
     private final boolean trackAcquiredPages;
 
+    /** Shared context. */
+    private final GridCacheSharedContext<?, ?> ctx;
+
     /**
      * @param log Logger.
      * @param directMemoryProvider Memory allocator to use.
@@ -184,6 +189,7 @@ public class PageMemoryNoStoreImpl implements PageMemory {
         this.trackAcquiredPages = trackAcquiredPages;
         this.memMetrics = memMetrics;
         this.dataRegionCfg = dataRegionCfg;
+        this.ctx = sharedCtx;
 
         sysPageSize = pageSize + PAGE_OVERHEAD;
 
@@ -288,8 +294,8 @@ public class PageMemoryNoStoreImpl implements PageMemory {
             }
         }
 
-        if (relPtr == INVALID_REL_PTR)
-            throw new IgniteOutOfMemoryException("Out of memory in data region 
[" +
+        if (relPtr == INVALID_REL_PTR) {
+            IgniteOutOfMemoryException oom = new 
IgniteOutOfMemoryException("Out of memory in data region [" +
                 "name=" + dataRegionCfg.getName() +
                 ", initSize=" + U.readableSize(dataRegionCfg.getInitialSize(), 
false) +
                 ", maxSize=" + U.readableSize(dataRegionCfg.getMaxSize(), 
false) +
@@ -299,6 +305,13 @@ public class PageMemoryNoStoreImpl implements PageMemory {
                 "  ^-- Enable eviction or expiration policies"
             );
 
+            if (ctx != null)
+                ctx.kernalContext().failure().process(new 
FailureContext(FailureType.CRITICAL_ERROR, oom));
+
+            throw oom;
+        }
+
+
         assert (relPtr & ~PageIdUtils.PAGE_IDX_MASK) == 0 : U.hexLong(relPtr & 
~PageIdUtils.PAGE_IDX_MASK);
 
         // Assign page ID according to flags and partition ID.

http://git-wip-us.apache.org/repos/asf/ignite/blob/9bb4ce8a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
----------------------------------------------------------------------
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index 46fb7dd..4463224 100755
--- 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -40,6 +40,8 @@ import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.configuration.DataRegionConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.mem.DirectMemoryProvider;
 import org.apache.ignite.internal.mem.DirectMemoryRegion;
@@ -543,7 +545,7 @@ public class PageMemoryImpl implements PageMemoryEx {
         catch (IgniteOutOfMemoryException oom) {
             DataRegionConfiguration dataRegionCfg = 
getDataRegionConfiguration();
 
-            throw (IgniteOutOfMemoryException) new 
IgniteOutOfMemoryException("Out of memory in data region [" +
+            IgniteOutOfMemoryException e = new IgniteOutOfMemoryException("Out 
of memory in data region [" +
                 "name=" + dataRegionCfg.getName() +
                 ", initSize=" + U.readableSize(dataRegionCfg.getInitialSize(), 
false) +
                 ", maxSize=" + U.readableSize(dataRegionCfg.getMaxSize(), 
false) +
@@ -551,7 +553,13 @@ public class PageMemoryImpl implements PageMemoryEx {
                 "  ^-- Increase maximum off-heap memory size 
(DataRegionConfiguration.maxSize)" + U.nl() +
                 "  ^-- Enable Ignite persistence 
(DataRegionConfiguration.persistenceEnabled)" + U.nl() +
                 "  ^-- Enable eviction or expiration policies"
-            ).initCause(oom);
+            );
+
+            e.initCause(oom);
+
+            ctx.kernalContext().failure().process(new 
FailureContext(FailureType.CRITICAL_ERROR, e));
+
+            throw e;
         }
         finally {
             seg.writeLock().unlock();
@@ -746,6 +754,11 @@ public class PageMemoryImpl implements PageMemoryEx {
 
             return absPtr;
         }
+        catch (IgniteOutOfMemoryException oom) {
+            ctx.kernalContext().failure().process(new 
FailureContext(FailureType.CRITICAL_ERROR, oom));
+
+            throw oom;
+        }
         finally {
             seg.writeLock().unlock();
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/9bb4ce8a/modules/core/src/test/java/org/apache/ignite/failure/AbstractFailureHandlerTest.java
----------------------------------------------------------------------
diff --git 
a/modules/core/src/test/java/org/apache/ignite/failure/AbstractFailureHandlerTest.java
 
b/modules/core/src/test/java/org/apache/ignite/failure/AbstractFailureHandlerTest.java
new file mode 100644
index 0000000..dc5f1f5
--- /dev/null
+++ 
b/modules/core/src/test/java/org/apache/ignite/failure/AbstractFailureHandlerTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.failure;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Abstract failure handler test.
+ */
+public class AbstractFailureHandlerTest extends GridCommonAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected FailureHandler getFailureHandler(String 
igniteInstanceName) {
+        return new DummyFailureHandler();
+    }
+
+    /**
+     * Gets dummy failure handler for ignite instance.
+     *
+     * @param ignite Ignite.
+     */
+    protected static DummyFailureHandler dummyFailureHandler(Ignite ignite) {
+        return (DummyFailureHandler)ignite.configuration().getFailureHandler();
+    }
+
+    /**
+     *
+     */
+    protected static class DummyFailureHandler implements FailureHandler {
+        /** Failure. */
+        private volatile boolean failure;
+
+        /** Failure context. */
+        private volatile FailureContext ctx;
+
+        /** {@inheritDoc} */
+        @Override public boolean onFailure(Ignite ignite, FailureContext 
failureCtx) {
+            failure = true;
+
+            ctx = failureCtx;
+
+            return true;
+        }
+
+        /**
+         * @return Failure.
+         */
+        public boolean failure() {
+            return failure;
+        }
+
+        /**
+         * @return Failure context.
+         */
+        public FailureContext failureContext() {
+            return ctx;
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/9bb4ce8a/modules/core/src/test/java/org/apache/ignite/failure/IoomFailureHandlerTest.java
----------------------------------------------------------------------
diff --git 
a/modules/core/src/test/java/org/apache/ignite/failure/IoomFailureHandlerTest.java
 
b/modules/core/src/test/java/org/apache/ignite/failure/IoomFailureHandlerTest.java
new file mode 100644
index 0000000..a777f81
--- /dev/null
+++ 
b/modules/core/src/test/java/org/apache/ignite/failure/IoomFailureHandlerTest.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.failure;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.transactions.Transaction;
+
+/**
+ * IgniteOutOfMemoryError failure handler test.
+ */
+public class IoomFailureHandlerTest extends AbstractFailureHandlerTest {
+    /** Offheap size for memory policy. */
+    private static final int SIZE = 10 * 1024 * 1024;
+
+    /** Page size. */
+    static final int PAGE_SIZE = 2048;
+
+    /** Number of entries. */
+    static final int ENTRIES = 10_000;
+
+    /** PDS enabled. */
+    private boolean pds;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String 
igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        DataStorageConfiguration dsCfg = new DataStorageConfiguration();
+
+        DataRegionConfiguration dfltPlcCfg = new DataRegionConfiguration();
+
+        dfltPlcCfg.setName("dfltPlc");
+        dfltPlcCfg.setInitialSize(SIZE);
+        dfltPlcCfg.setMaxSize(SIZE);
+
+        if (pds)
+            dfltPlcCfg.setPersistenceEnabled(true);
+
+        dsCfg.setDefaultDataRegionConfiguration(dfltPlcCfg);
+        dsCfg.setPageSize(PAGE_SIZE);
+
+        cfg.setDataStorageConfiguration(dsCfg);
+
+        CacheConfiguration<?, ?> ccfg = new CacheConfiguration<>()
+            .setName(DEFAULT_CACHE_NAME)
+            .setCacheMode(CacheMode.PARTITIONED)
+            .setBackups(0)
+            .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+
+        cfg.setCacheConfiguration(ccfg);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        cleanPersistenceDir();
+
+        super.beforeTest();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        cleanPersistenceDir();
+    }
+
+    /**
+     * Test IgniteOutOfMemoryException handling with no store.
+     */
+    public void testIoomErrorNoStoreHandling() throws Exception {
+        testIoomErrorHandling(false);
+    }
+
+    /**
+     * Test IgniteOutOfMemoryException handling with PDS.
+     */
+    public void testIoomErrorPdsHandling() throws Exception {
+        testIoomErrorHandling(true);
+    }
+
+    /**
+     * Test IOOME handling.
+     */
+    public void testIoomErrorHandling(boolean pds) throws Exception {
+        this.pds = pds;
+
+        IgniteEx ignite0 = startGrid(0);
+        IgniteEx ignite1 = startGrid(1);
+
+        try {
+            if (pds)
+                ignite0.cluster().active(true);
+
+            IgniteCache<Integer, Object> cache0 = 
ignite0.getOrCreateCache(DEFAULT_CACHE_NAME);
+            IgniteCache<Integer, Object> cache1 = 
ignite1.getOrCreateCache(DEFAULT_CACHE_NAME);
+
+            awaitPartitionMapExchange();
+
+            try (Transaction tx = ignite0.transactions().txStart()) {
+                for (Integer i : primaryKeys(cache1, ENTRIES))
+                    cache0.put(i, new byte[PAGE_SIZE / 3 * 2]);
+
+                tx.commit();
+            }
+            catch (Throwable ignore) {
+                // Expected.
+            }
+
+            assertFalse(dummyFailureHandler(ignite0).failure());
+            assertTrue(dummyFailureHandler(ignite1).failure());
+            
assertTrue(X.hasCause(dummyFailureHandler(ignite1).failureContext().error(), 
IgniteOutOfMemoryException.class));
+        }
+        finally {
+            stopGrid(1);
+            stopGrid(0);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/9bb4ce8a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
----------------------------------------------------------------------
diff --git 
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
 
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
index 31af118..3697c4c 100644
--- 
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
+++ 
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
@@ -28,6 +28,7 @@ import java.util.concurrent.atomic.AtomicReference;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.NoOpFailureHandler;
 import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
 import org.apache.ignite.internal.mem.DirectMemoryProvider;
 import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
@@ -41,6 +42,7 @@ import 
org.apache.ignite.internal.processors.cache.persistence.CheckpointWritePr
 import 
org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
 import 
org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.processors.failure.FailureProcessor;
 import org.apache.ignite.internal.processors.plugin.IgnitePluginProcessor;
 import org.apache.ignite.internal.util.lang.GridInClosure3X;
 import org.apache.ignite.plugin.PluginProvider;
@@ -268,10 +270,17 @@ public class PageMemoryImplTest extends 
GridCommonAbstractTest {
 
         IgniteConfiguration igniteCfg = new IgniteConfiguration();
         igniteCfg.setDataStorageConfiguration(new DataStorageConfiguration());
+        igniteCfg.setFailureHandler(new NoOpFailureHandler());
 
         GridTestKernalContext kernalCtx = new GridTestKernalContext(new 
GridTestLog4jLogger(), igniteCfg);
         kernalCtx.add(new IgnitePluginProcessor(kernalCtx, igniteCfg, 
Collections.<PluginProvider>emptyList()));
 
+        FailureProcessor failureProc = new FailureProcessor(kernalCtx);
+
+        failureProc.start();
+
+        kernalCtx.add(failureProc);
+
         GridCacheSharedContext<Object, Object> sharedCtx = new 
GridCacheSharedContext<>(
             kernalCtx,
             null,

http://git-wip-us.apache.org/repos/asf/ignite/blob/9bb4ce8a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
----------------------------------------------------------------------
diff --git 
a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
 
b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
index dd9cdfd..c4b7d92 100644
--- 
a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
+++ 
b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
@@ -21,6 +21,7 @@ import java.util.Set;
 import junit.framework.TestSuite;
 import org.apache.ignite.GridSuppressedExceptionSelfTest;
 import org.apache.ignite.failure.FailureHandlerTriggeredTest;
+import org.apache.ignite.failure.IoomFailureHandlerTest;
 import org.apache.ignite.failure.StopNodeFailureHandlerTest;
 import org.apache.ignite.failure.StopNodeOrHaltFailureHandlerTest;
 import org.apache.ignite.internal.ClassSetTest;
@@ -197,6 +198,7 @@ public class IgniteBasicTestSuite extends TestSuite {
         suite.addTestSuite(FailureHandlerTriggeredTest.class);
         suite.addTestSuite(StopNodeFailureHandlerTest.class);
         suite.addTestSuite(StopNodeOrHaltFailureHandlerTest.class);
+        suite.addTestSuite(IoomFailureHandlerTest.class);
 
         return suite;
     }

Reply via email to