This is an automated email from the ASF dual-hosted git repository. amashenkov pushed a commit to branch gg-19225 in repository https://gitbox.apache.org/repos/asf/ignite.git
commit 9d9c21af5577189df18ba937f0013a81f9001745 Author: mstepachev <[email protected]> AuthorDate: Wed May 29 17:10:50 2019 +0300 GG-19066 [IGNITE-10983] Check that persistenceEnabled is consistent on all nodes. --- .../processors/cache/GridCacheProcessor.java | 92 ++++- .../internal/processors/cache/GridCacheUtils.java | 38 ++ ...heWithDifferentDataRegionConfigurationTest.java | 409 +++++++++++++++++++++ .../ignite/testsuites/IgniteCacheTestSuite.java | 2 + 4 files changed, 540 insertions(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index b623801..cb82a37 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -28,6 +28,7 @@ import java.util.LinkedList; import java.util.List; import java.util.ListIterator; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -35,6 +36,7 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.cache.configuration.FactoryBuilder; import javax.cache.expiry.EternalExpiryPolicy; import javax.cache.expiry.ExpiryPolicy; @@ -56,6 +58,7 @@ import org.apache.ignite.cluster.ClusterGroup; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.DataPageEvictionMode; +import org.apache.ignite.configuration.DataRegionConfiguration; import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.DeploymentMode; import org.apache.ignite.configuration.FileSystemConfiguration; @@ -207,6 +210,7 @@ import static org.apache.ignite.internal.IgniteComponentType.JTA; import static org.apache.ignite.internal.IgniteFeatures.TRANSACTION_OWNER_THREAD_DUMP_PROVIDING; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_CONSISTENCY_CHECK_SKIPPED; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_TX_CONFIG; +import static org.apache.ignite.internal.processors.cache.GridCacheUtils.isDefaultDataRegionPersistent; import static org.apache.ignite.internal.processors.cache.GridCacheUtils.isNearEnabled; import static org.apache.ignite.internal.processors.cache.GridCacheUtils.isPersistentCache; import static org.apache.ignite.internal.util.IgniteUtils.doInParallel; @@ -224,6 +228,11 @@ public class GridCacheProcessor extends GridProcessorAdapter { private static final String MERGE_OF_CONFIG_REQUIRED_MESSAGE = "Failed to join node to the active cluster " + "(the config of the cache '%s' has to be merged which is impossible on active grid). " + "Deactivate grid and retry node join or clean the joining node."; + + /** Invalid region configuration message. */ + private static final String INVALID_REGION_CONFIGURATION_MESSAGE = "Failed to join node " + + "(Incompatible data region configuration [region=%s, locNodeId=%s, isPersistenceEnabled=%s, rmtNodeId=%s, isPersistenceEnabled=%s])"; + /** */ private final boolean startClientCaches = IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_START_CACHES_ON_JOIN, false); @@ -2799,7 +2808,7 @@ public class GridCacheProcessor extends GridProcessorAdapter { String memPlcName = cfg.getDataRegionName(); - DataRegion dataRegion = sharedCtx.database().dataRegion(memPlcName); + DataRegion dataRegion = affNode ? sharedCtx.database().dataRegion(memPlcName) : null; FreeList freeList = sharedCtx.database().freeList(memPlcName); ReuseList reuseList = sharedCtx.database().reuseList(memPlcName); @@ -3127,6 +3136,32 @@ public class GridCacheProcessor extends GridProcessorAdapter { } /** + * @param node Remote node to check. + * @return Data storage configuration + */ + private DataStorageConfiguration extractDataStorage(ClusterNode rmtNode) { + return GridCacheUtils.extractDataStorage( + rmtNode, + ctx.marshallerContext().jdkMarshaller(), + U.resolveClassLoader(ctx.config()) + ); + } + + /** + * @param dataStorageCfg User-defined data regions. + */ + private Map<String, DataRegionConfiguration> dataRegionCfgs(DataStorageConfiguration dataStorageCfg) { + if (dataStorageCfg != null) { + return Optional.ofNullable(dataStorageCfg.getDataRegionConfigurations()) + .map(Stream::of) + .orElseGet(Stream::empty) + .collect(Collectors.toMap(DataRegionConfiguration::getName, e -> e)); + } + + return Collections.emptyMap(); + } + + /** * Force checkpoint and remove offheap checkpoint listener after it was finished. * * @param grpToStop Cache group to stop. @@ -3417,6 +3452,15 @@ public class GridCacheProcessor extends GridProcessorAdapter { StringBuilder errorMessage = new StringBuilder(); + if (!node.isClient()) { + validateRmtRegions(node).forEach(error -> { + if (errorMessage.length() > 0) + errorMessage.append("\n"); + + errorMessage.append(error); + }); + } + for (CacheJoinNodeDiscoveryData.CacheInfo cacheInfo : nodeData.caches().values()) { try { byte[] secCtxBytes = node.attribute(IgniteNodeAttributes.ATTR_SECURITY_SUBJECT_V2); @@ -3534,6 +3578,52 @@ public class GridCacheProcessor extends GridProcessorAdapter { } /** + * @param rmtNode Joining node. + * @return List of validation errors. + */ + private List<String> validateRmtRegions(ClusterNode rmtNode) { + List<String> errorMessages = new ArrayList<>(); + + DataStorageConfiguration rmtStorageCfg = extractDataStorage(rmtNode); + Map<String, DataRegionConfiguration> rmtRegionCfgs = dataRegionCfgs(rmtStorageCfg); + + DataStorageConfiguration locStorageCfg = ctx.config().getDataStorageConfiguration(); + + if (isDefaultDataRegionPersistent(locStorageCfg) != isDefaultDataRegionPersistent(rmtStorageCfg)) { + errorMessages.add(String.format( + INVALID_REGION_CONFIGURATION_MESSAGE, + "DEFAULT", + ctx.localNodeId(), + isDefaultDataRegionPersistent(locStorageCfg), + rmtNode.id(), + isDefaultDataRegionPersistent(rmtStorageCfg) + )); + } + + for (ClusterNode clusterNode : ctx.discovery().aliveServerNodes()) { + Map<String, DataRegionConfiguration> nodeRegionCfg = dataRegionCfgs(extractDataStorage(clusterNode)); + + for (Map.Entry<String, DataRegionConfiguration> nodeRegionCfgEntry : nodeRegionCfg.entrySet()) { + String regionName = nodeRegionCfgEntry.getKey(); + + DataRegionConfiguration rmtRegionCfg = rmtRegionCfgs.get(regionName); + + if (rmtRegionCfg != null && rmtRegionCfg.isPersistenceEnabled() != nodeRegionCfgEntry.getValue().isPersistenceEnabled()) + errorMessages.add(String.format( + INVALID_REGION_CONFIGURATION_MESSAGE, + regionName, + ctx.localNodeId(), + nodeRegionCfgEntry.getValue().isPersistenceEnabled(), + rmtNode.id(), + rmtRegionCfg.isPersistenceEnabled() + )); + } + } + + return errorMessages; + } + + /** * Cache statistics clear message received. * * @param msg Message. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java index 0d8acd2..a2031a3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java @@ -98,6 +98,7 @@ import org.apache.ignite.lang.IgniteInClosure; import org.apache.ignite.lang.IgnitePredicate; import org.apache.ignite.lang.IgniteReducer; import org.apache.ignite.lifecycle.LifecycleAware; +import org.apache.ignite.marshaller.jdk.JdkMarshaller; import org.apache.ignite.plugin.CachePluginConfiguration; import org.apache.ignite.plugin.security.SecurityException; import org.apache.ignite.spi.encryption.EncryptionSpi; @@ -1873,6 +1874,43 @@ public class GridCacheUtils { } /** + * @return {@code true} if persistence is enabled for a default data region, {@code false} if not. + */ + public static boolean isDefaultDataRegionPersistent(DataStorageConfiguration cfg) { + if (cfg == null) + return false; + + DataRegionConfiguration dfltRegionCfg = cfg.getDefaultDataRegionConfiguration(); + + if (dfltRegionCfg == null) + return false; + + return dfltRegionCfg.isPersistenceEnabled(); + } + + /** + * Extract and unmarshal data storage configuration from given node. + * + * @param node Source of data storage configuration. + * @return Data storage configuration for given node, or {@code null} if this node has not data storage + * configuration. + */ + @Nullable public static DataStorageConfiguration extractDataStorage(ClusterNode node, JdkMarshaller marshaller, + ClassLoader clsLdr) { + Object dsCfgBytes = node.attribute(IgniteNodeAttributes.ATTR_DATA_STORAGE_CONFIG); + + if (dsCfgBytes instanceof byte[]) { + try { + return marshaller.unmarshal((byte[])dsCfgBytes, clsLdr); + } + catch (IgniteCheckedException e) { + throw new IgniteException(e); + } + } + return null; + } + + /** * @return {@code true} if persistence is enabled for at least one data region, {@code false} if not. */ public static boolean isPersistenceEnabled(DataStorageConfiguration cfg) { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheWithDifferentDataRegionConfigurationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheWithDifferentDataRegionConfigurationTest.java new file mode 100644 index 0000000..059b77f --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheWithDifferentDataRegionConfigurationTest.java @@ -0,0 +1,409 @@ +/* + * Copyright 2019 GridGain Systems, Inc. and Contributors. + * + * Licensed under the GridGain Community Edition License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.gridgain.com/products/software/community-edition/gridgain-community-edition-license + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.util.typedef.X; +import org.apache.ignite.lang.IgnitePredicate; +import org.apache.ignite.spi.IgniteSpiException; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.jetbrains.annotations.Nullable; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.apache.ignite.testframework.GridTestUtils.assertThrowsWithCause; + +/** + * Data regions validation test on joining node. + */ +public class CacheWithDifferentDataRegionConfigurationTest extends GridCommonAbstractTest { + /** Node 1. */ + private static final int NODE_1 = 0; + + /** Node 2. */ + private static final int NODE_2 = 1; + + /** Node 3. */ + private static final int NODE_3 = 2; + + /** Region 1. */ + private static final String REGION_1 = "region_1"; + + /** Region 2. */ + private static final String REGION_2 = "region_2"; + + /** Region 3. */ + private static final String REGION_3 = "region_3"; + + /** Region 4. */ + private static final String REGION_4 = "region_4"; + + /** Cache 1. */ + private static final String CACHE_1 = "cache_1"; + + /** Cache 2. */ + private static final String CACHE_2 = "cache_2"; + + /** Persistence. */ + private static final boolean PERSISTENCE = true; + + /** Memory. */ + private static final boolean MEMORY = false; + + /** + * @throws Exception If failed. + */ + @After + public void tearDown() throws Exception { + stopAllGrids(); + cleanPersistenceDir(); + } + + /** + * @throws Exception If failed. + */ + @Before + public void setUp() throws Exception { + super.setUp(); + cleanPersistenceDir(); + } + + /** + * + */ + @Test + public void twoNodesHaveDifferentDefaultConfigurationUnacceptable() throws Exception { + node(NODE_1) + .withDefaultRegion("defaultName1", MEMORY) + .andCache(CACHE_1) + .start(); + + assertThrowsContainsMessage(() -> node(NODE_2).withDefaultRegion("defaultName2", PERSISTENCE).andCache(CACHE_2).start(), + IgniteSpiException.class, + "Failed to join node (Incompatible data region configuration [region=DEFAULT" + ); + } + + /** + * + */ + @Test + public void twoNodesHaveCommonDefaultConfigurationAcceptable() throws Exception { + IgniteEx node1 = node(NODE_1) + .withDefaultRegion("defaultName1", PERSISTENCE) + .andCache(CACHE_1) + .start(); + + IgniteEx node2 = node(NODE_2) + .withDefaultRegion("defaultName2", PERSISTENCE) + .andCache(CACHE_2) + .start(); + + node1.cluster().active(true); + + populateCache(node1, CACHE_1, 1000); + populateCache(node2, CACHE_2, 350); + + assertThatCacheContains(node2, CACHE_1, 1000); + assertThatCacheContains(node1, CACHE_2, 350); + } + + /** + * + */ + @Test + public void firstNodeHasDefaultAndSecondDefaultWithCustomNameAcceptable() throws Exception { + IgniteEx node1 = node(NODE_1) + .andCache(CACHE_1) + .start(); + + IgniteEx node2 = node(NODE_2) + .withDefaultRegion("defaultName2", MEMORY) + .andCache(CACHE_2) + .start(); + + node1.cluster().active(true); + + populateCache(node1, CACHE_1, 1000); + populateCache(node2, CACHE_2, 350); + + assertThatCacheContains(node2, CACHE_1, 1000); + assertThatCacheContains(node1, CACHE_2, 350); + } + + /** + * + */ + @Test + public void firstNodeHasDefaultAndSecondWithTwoRegionsDefaultAndPersistenceAcceptable() throws Exception { + IgniteEx node1 = node(NODE_1) + .andCache(CACHE_1) + .start(); + + IgniteEx node2 = node(NODE_2) + .withRegion(REGION_1, MEMORY) + .withRegion(REGION_2, PERSISTENCE) + .andExclusiveCache(CACHE_2, REGION_2) + .start(); + + node1.cluster().active(true); + + populateCache(node1, CACHE_1, 1000); + populateCache(node2, CACHE_2, 350); + + assertThatCacheContains(node2, CACHE_1, 1000); + assertThatCacheContains(node1, CACHE_2, 350); + } + + /** + * + */ + @Test + public void twoNodesHaveTwoNonOverlappingRegionsAcceptable() throws Exception { + IgniteEx node1 = node(NODE_1) + .withRegion(REGION_1, PERSISTENCE) + .withRegion(REGION_2, MEMORY) + .andExclusiveCache(CACHE_1, REGION_1) + .start(); + + IgniteEx node2 = node(NODE_2) + .withRegion(REGION_3, MEMORY) + .withRegion(REGION_4, PERSISTENCE) + .andExclusiveCache(CACHE_2, REGION_4) + .start(); + + node1.cluster().active(true); + + populateCache(node1, CACHE_1, 1000); + populateCache(node2, CACHE_2, 350); + + assertThatCacheContains(node2, CACHE_1, 1000); + assertThatCacheContains(node1, CACHE_2, 350); + } + + /** + * + */ + @Test + public void twoNodesWithSameRegionsButDifferentPersistenceModeForThemUnacceptable() throws Exception { + node(NODE_1) + .withRegion(REGION_1, PERSISTENCE) + .start(); + + assertThrowsContainsMessage(() -> node(NODE_2).withRegion(REGION_1, MEMORY).start(), + IgniteSpiException.class, + "Failed to join node (Incompatible data region configuration [region=" + REGION_1 + ); + } + + /** + * + */ + @Test + public void secondNodeMustRejectJoinOnThirdNode() throws Exception { + node(NODE_1) + .start(); + + node(NODE_2) + .withRegion(REGION_2, PERSISTENCE) + .start(); + + assertThrowsContainsMessage(() -> node(NODE_3).withRegion(REGION_2, MEMORY).start(), + IgniteSpiException.class, + "Failed to join node (Incompatible data region configuration [region=" + REGION_2 + ); + } + + /** + * @param call Callable. + * @param cls Class. + * @param msg Message. + */ + private void assertThrowsContainsMessage(Callable<?> call, Class<? extends Throwable> cls, String msg) { + Throwable throwable = assertThrowsWithCause(call, cls); + + assertTrue("Message mismatch: " + msg, X.hasCause(throwable, msg, cls)); + } + + /** + * @param node Node. + * @param cacheName Cache name. + * @param size Size. + */ + private void assertThatCacheContains(IgniteEx node, String cacheName, int size) { + IgniteCache<Integer, Integer> cache = node.getOrCreateCache(cacheName); + + for (int i = 0; i < size; i++) + assertEquals((Integer)i, cache.get(i)); + } + + /** + * @param node Node. + * @param cacheName Cache name. + * @param size Size. + */ + public void populateCache(IgniteEx node, String cacheName, int size) { + IgniteCache<Integer, Integer> cache = node.getOrCreateCache(cacheName); + + for (int i = 0; i < size; i++) + cache.put(i, i); + } + + /** + * @param gridId Grid id. + */ + private ConfigurationBuilder node(int gridId) { + return new ConfigurationBuilder(gridId); + } + + /** + * + */ + private static class NodeFilter implements IgnitePredicate<ClusterNode> { + /** + * + */ + private final String consistenceId; + + /** + * + */ + private NodeFilter(String consistenceId) { + this.consistenceId = consistenceId; + } + + /** {@inheritDoc} */ + @Override public boolean apply(ClusterNode clusterNode) { + + return clusterNode.consistentId().equals(consistenceId); + } + } + + /** + * + */ + private class ConfigurationBuilder { + /** Grid id. */ + private final String gridName; + + /** Regions. */ + private final List<DataRegionConfiguration> regions = new ArrayList<>(); + + /** Caches. */ + private final List<CacheConfiguration> caches = new ArrayList<>(); + + /** Default region configuration. */ + @Nullable private DataRegionConfiguration dfltRegionConfiguration; + + /** + * @param gridId Grid id. + */ + ConfigurationBuilder(int gridId) { + this.gridName = getTestIgniteInstanceName(gridId); + } + + /** + * @param regionName Region name. + * @param persistence Persistence. + */ + ConfigurationBuilder withDefaultRegion(String regionName, boolean persistence) { + dfltRegionConfiguration = new DataRegionConfiguration() + .setName(regionName) + .setInitialSize(100L * 1024 * 1024) + .setMaxSize(500L * 1024 * 1024) + .setPersistenceEnabled(persistence); + + return this; + } + + /** + * @param regionName Region name. + * @param persistence Persistence. + */ + ConfigurationBuilder withRegion(String regionName, boolean persistence) { + regions.add(new DataRegionConfiguration() + .setName(regionName) + .setInitialSize(100L * 1024 * 1024) + .setMaxSize(500L * 1024 * 1024) + .setPersistenceEnabled(persistence) + ); + + return this; + } + + /** + * @param cacheName Cache name. + */ + ConfigurationBuilder andCache(String cacheName) { + return andCache(cacheName, null); + } + + /** + * @param cacheName Cache name. + * @param regionName Region name. + */ + ConfigurationBuilder andCache(String cacheName, String regionName) { + caches.add(new CacheConfiguration().setDataRegionName(regionName).setName(cacheName)); + + return this; + } + + /** + * This cache related with node via node filter. + * + * @param cacheName Cache name. + * @param regionName Region name. + */ + ConfigurationBuilder andExclusiveCache(String cacheName, String regionName) { + caches.add(new CacheConfiguration() + .setNodeFilter(new NodeFilter(gridName)) + .setDataRegionName(regionName) + .setName(cacheName) + ); + + return this; + } + + /** Start node from builder */ + public IgniteEx start() throws Exception { + IgniteConfiguration cfg = getConfiguration(gridName); + + cfg.setConsistentId(gridName); + + DataStorageConfiguration storageCfg = new DataStorageConfiguration(); + storageCfg.setDataRegionConfigurations(regions.toArray(new DataRegionConfiguration[regions.size()])); + cfg.setDataStorageConfiguration(storageCfg); + + if (dfltRegionConfiguration != null) + storageCfg.setDefaultDataRegionConfiguration(dfltRegionConfiguration); + + cfg.setCacheConfiguration(caches.toArray(new CacheConfiguration[caches.size()])); + + return startGrid(cfg); + } + } +} \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java index 8d48ad4..7232874 100755 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java @@ -52,6 +52,7 @@ import org.apache.ignite.internal.processors.cache.CacheNamesSelfTest; import org.apache.ignite.internal.processors.cache.CacheNamesWithSpecialCharactersTest; import org.apache.ignite.internal.processors.cache.CachePutEventListenerErrorSelfTest; import org.apache.ignite.internal.processors.cache.CacheTxFastFinishTest; +import org.apache.ignite.internal.processors.cache.CacheWithDifferentDataRegionConfigurationTest; import org.apache.ignite.internal.processors.cache.DataStorageConfigurationValidationTest; import org.apache.ignite.internal.processors.cache.GridCacheAffinityApiSelfTest; import org.apache.ignite.internal.processors.cache.GridCacheAffinityMapperSelfTest; @@ -231,6 +232,7 @@ public class IgniteCacheTestSuite { GridTestUtils.addTestIfNeeded(suite, GridCacheConfigurationValidationSelfTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, GridCacheConfigurationConsistencySelfTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, GridDataStorageConfigurationConsistencySelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheWithDifferentDataRegionConfigurationTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, DataStorageConfigurationValidationTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, GridCacheJdbcBlobStoreSelfTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, GridCacheJdbcBlobStoreMultithreadedSelfTest.class, ignoredTests);
