Repository: ignite Updated Branches: refs/heads/master 5a2ef66cb -> 76ad0a4d1
IGNITE-10003 Introduced SYSTEM_CRITICAL_OPERATION_TIMEOUT failure type Signed-off-by: Andrey Gura <ag...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/76ad0a4d Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/76ad0a4d Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/76ad0a4d Branch: refs/heads/master Commit: 76ad0a4d10ac761f5cdec01e6ec4d7299e434a07 Parents: 5a2ef66 Author: Andrey Kuznetsov <stku...@gmail.com> Authored: Mon Dec 24 17:38:37 2018 +0300 Committer: Andrey Gura <ag...@apache.org> Committed: Mon Dec 24 17:38:37 2018 +0300 ---------------------------------------------------------------------- .../ignite/failure/AbstractFailureHandler.java | 5 +- .../org/apache/ignite/failure/FailureType.java | 5 +- .../GridCacheDatabaseSharedManager.java | 7 +- .../failure/SystemWorkersBlockingTest.java | 17 ++- .../CheckpointReadLockFailureTest.java | 125 +++++++++++++++++++ .../IgniteBasicWithPersistenceTestSuite.java | 6 +- 6 files changed, 156 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/76ad0a4d/modules/core/src/main/java/org/apache/ignite/failure/AbstractFailureHandler.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/failure/AbstractFailureHandler.java b/modules/core/src/main/java/org/apache/ignite/failure/AbstractFailureHandler.java index 285c49d..5e69161 100644 --- a/modules/core/src/main/java/org/apache/ignite/failure/AbstractFailureHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/failure/AbstractFailureHandler.java @@ -18,11 +18,13 @@ package org.apache.ignite.failure; import java.util.Collections; +import java.util.EnumSet; import java.util.Set; import org.apache.ignite.Ignite; import org.apache.ignite.internal.util.tostring.GridToStringInclude; import org.apache.ignite.internal.util.typedef.internal.S; +import static org.apache.ignite.failure.FailureType.SYSTEM_CRITICAL_OPERATION_TIMEOUT; import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_BLOCKED; /** @@ -33,7 +35,8 @@ import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_BLOCKED; public abstract class AbstractFailureHandler implements FailureHandler { /** */ @GridToStringInclude - private Set<FailureType> ignoredFailureTypes = Collections.singleton(SYSTEM_WORKER_BLOCKED); + private Set<FailureType> ignoredFailureTypes = + Collections.unmodifiableSet(EnumSet.of(SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT)); /** * Sets failure types that must be ignored by failure handler. http://git-wip-us.apache.org/repos/asf/ignite/blob/76ad0a4d/modules/core/src/main/java/org/apache/ignite/failure/FailureType.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/failure/FailureType.java b/modules/core/src/main/java/org/apache/ignite/failure/FailureType.java index fbd5529f..114e432 100644 --- a/modules/core/src/main/java/org/apache/ignite/failure/FailureType.java +++ b/modules/core/src/main/java/org/apache/ignite/failure/FailureType.java @@ -31,5 +31,8 @@ public enum FailureType { SYSTEM_WORKER_BLOCKED, /** Critical error - error which leads to the system's inoperability. */ - CRITICAL_ERROR + CRITICAL_ERROR, + + /** System-critical operation has been timed out. */ + SYSTEM_CRITICAL_OPERATION_TIMEOUT } http://git-wip-us.apache.org/repos/asf/ignite/blob/76ad0a4d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index 34c3b72..62e5186 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -175,10 +175,11 @@ import static java.nio.file.StandardOpenOption.READ; import static org.apache.ignite.IgniteSystemProperties.IGNITE_CHECKPOINT_READ_LOCK_TIMEOUT; import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD; import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR; +import static org.apache.ignite.failure.FailureType.SYSTEM_CRITICAL_OPERATION_TIMEOUT; import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION; import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.CHECKPOINT_RECORD; -import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.TMP_FILE_MATCHER; import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.METASTORE_DATA_RECORD; +import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.TMP_FILE_MATCHER; import static org.apache.ignite.internal.util.IgniteUtils.checkpointBufferSize; /** @@ -266,7 +267,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan private volatile GridFutureAdapter<Void> enableChangeApplied; /** */ - private ReentrantReadWriteLock checkpointLock = new ReentrantReadWriteLock(); + ReentrantReadWriteLock checkpointLock = new ReentrantReadWriteLock(); /** */ private long checkpointFreq; @@ -1569,7 +1570,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan IgniteException e = new IgniteException(msg); - if (cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, e))) + if (cctx.kernalContext().failure().process(new FailureContext(SYSTEM_CRITICAL_OPERATION_TIMEOUT, e))) throw e; throw new CheckpointReadLockTimeoutException(msg); http://git-wip-us.apache.org/repos/asf/ignite/blob/76ad0a4d/modules/core/src/test/java/org/apache/ignite/failure/SystemWorkersBlockingTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/failure/SystemWorkersBlockingTest.java b/modules/core/src/test/java/org/apache/ignite/failure/SystemWorkersBlockingTest.java index a228126..3d7e5df 100644 --- a/modules/core/src/test/java/org/apache/ignite/failure/SystemWorkersBlockingTest.java +++ b/modules/core/src/test/java/org/apache/ignite/failure/SystemWorkersBlockingTest.java @@ -17,6 +17,8 @@ package org.apache.ignite.failure; +import java.util.HashSet; +import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import org.apache.ignite.Ignite; @@ -44,13 +46,22 @@ public class SystemWorkersBlockingTest extends GridCommonAbstractTest { @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - cfg.setFailureHandler(new AbstractFailureHandler() { + AbstractFailureHandler failureHnd = new AbstractFailureHandler() { @Override protected boolean handle(Ignite ignite, FailureContext failureCtx) { - hndLatch.countDown(); + if (failureCtx.type() == FailureType.SYSTEM_WORKER_BLOCKED) + hndLatch.countDown(); return false; } - }); + }; + + Set<FailureType> ignoredFailureTypes = new HashSet<>(failureHnd.getIgnoredFailureTypes()); + + ignoredFailureTypes.remove(FailureType.SYSTEM_WORKER_BLOCKED); + + failureHnd.setIgnoredFailureTypes(ignoredFailureTypes); + + cfg.setFailureHandler(failureHnd); cfg.setFailureDetectionTimeout(FAILURE_DETECTION_TIMEOUT); http://git-wip-us.apache.org/repos/asf/ignite/blob/76ad0a4d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/CheckpointReadLockFailureTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/CheckpointReadLockFailureTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/CheckpointReadLockFailureTest.java new file mode 100644 index 0000000..2ec5c2d --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/CheckpointReadLockFailureTest.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence; + +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.apache.ignite.Ignite; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.failure.AbstractFailureHandler; +import org.apache.ignite.failure.FailureContext; +import org.apache.ignite.failure.FailureType; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +/** + * Tests critical failure handling on checkpoint read lock acquisition errors. + */ +public class CheckpointReadLockFailureTest extends GridCommonAbstractTest { + /** */ + private static final AbstractFailureHandler FAILURE_HND = new AbstractFailureHandler() { + @Override protected boolean handle(Ignite ignite, FailureContext failureCtx) { + if (failureCtx.type() != FailureType.SYSTEM_CRITICAL_OPERATION_TIMEOUT) + return true; + + if (hndLatch != null) + hndLatch.countDown(); + + return false; + } + }; + + /** */ + private static volatile CountDownLatch hndLatch; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + return super.getConfiguration(igniteInstanceName) + .setFailureHandler(FAILURE_HND) + .setDataStorageConfiguration(new DataStorageConfiguration() + .setDefaultDataRegionConfiguration(new DataRegionConfiguration() + .setPersistenceEnabled(true)) + .setCheckpointFrequency(Integer.MAX_VALUE) + .setCheckpointReadLockTimeout(1)); + } + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + Set<FailureType> ignoredFailureTypes = new HashSet<>(FAILURE_HND.getIgnoredFailureTypes()); + ignoredFailureTypes.remove(FailureType.SYSTEM_CRITICAL_OPERATION_TIMEOUT); + + FAILURE_HND.setIgnoredFailureTypes(ignoredFailureTypes); + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + cleanPersistenceDir(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + cleanPersistenceDir(); + } + + /** + * @throws Exception If failed. + */ + public void testFailureTypeOnTimeout() throws Exception { + hndLatch = new CountDownLatch(1); + + IgniteEx ig = startGrid(0); + + ig.cluster().active(true); + + GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)ig.context().cache().context().database(); + + IgniteInternalFuture acquireWriteLock = GridTestUtils.runAsync(() -> { + db.checkpointLock.writeLock().lock(); + + try { + doSleep(Long.MAX_VALUE); + } + finally { + db.checkpointLock.writeLock().unlock(); + } + }); + + GridTestUtils.waitForCondition(() -> db.checkpointLock.writeLock().isHeldByCurrentThread(), 5000); + + IgniteInternalFuture acquireReadLock = GridTestUtils.runAsync(() -> { + db.checkpointReadLock(); + db.checkpointReadUnlock(); + }); + + assertTrue(hndLatch.await(5, TimeUnit.SECONDS)); + + acquireWriteLock.cancel(); + + acquireReadLock.get(5, TimeUnit.SECONDS); + + GridTestUtils.waitForCondition(acquireWriteLock::isCancelled, 5000); + + stopGrid(0); + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/76ad0a4d/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java index 4dbcf43..eefce7f 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java @@ -22,8 +22,10 @@ import junit.framework.JUnit4TestAdapter; import junit.framework.TestSuite; import org.apache.ignite.failure.FailureHandlingConfigurationTest; import org.apache.ignite.failure.IoomFailureHandlerTest; +import org.apache.ignite.failure.SystemWorkersBlockingTest; import org.apache.ignite.failure.SystemWorkersTerminationTest; import org.apache.ignite.internal.ClusterBaselineNodesMetricsSelfTest; +import org.apache.ignite.internal.GridNodeMetricsLogPdsSelfTest; import org.apache.ignite.internal.encryption.EncryptedCacheBigEntryTest; import org.apache.ignite.internal.encryption.EncryptedCacheCreateTest; import org.apache.ignite.internal.encryption.EncryptedCacheDestroyTest; @@ -31,7 +33,7 @@ import org.apache.ignite.internal.encryption.EncryptedCacheGroupCreateTest; import org.apache.ignite.internal.encryption.EncryptedCacheNodeJoinTest; import org.apache.ignite.internal.encryption.EncryptedCachePreconfiguredRestartTest; import org.apache.ignite.internal.encryption.EncryptedCacheRestartTest; -import org.apache.ignite.internal.GridNodeMetricsLogPdsSelfTest; +import org.apache.ignite.internal.processors.cache.persistence.CheckpointReadLockFailureTest; import org.apache.ignite.internal.processors.service.ServiceDeploymentOnActivationTest; import org.apache.ignite.internal.processors.service.ServiceDeploymentOutsideBaselineTest; import org.apache.ignite.marshaller.GridMarshallerMappingConsistencyTest; @@ -68,6 +70,8 @@ public class IgniteBasicWithPersistenceTestSuite { suite.addTest(new JUnit4TestAdapter(GridMarshallerMappingConsistencyTest.class)); suite.addTest(new JUnit4TestAdapter(SystemWorkersTerminationTest.class)); suite.addTest(new JUnit4TestAdapter(FailureHandlingConfigurationTest.class)); + suite.addTest(new JUnit4TestAdapter(SystemWorkersBlockingTest.class)); + suite.addTest(new JUnit4TestAdapter(CheckpointReadLockFailureTest.class)); suite.addTest(new JUnit4TestAdapter(GridCommandHandlerTest.class)); suite.addTest(new JUnit4TestAdapter(GridCommandHandlerSslTest.class));
