This is an automated email from the ASF dual-hosted git repository. irakov pushed a commit to branch ignite-2.8 in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/ignite-2.8 by this push: new 6892c66 IGNITE-12531 Cluster is unable to change BLT on 2.8 if storage was initially created on 2.7 or less - Fixes #7265. 6892c66 is described below commit 6892c6639487211aadb1ea47893a017b636d6f9d Author: Slava Koptilin <slava.kopti...@gmail.com> AuthorDate: Fri Jan 17 18:31:41 2020 +0300 IGNITE-12531 Cluster is unable to change BLT on 2.8 if storage was initially created on 2.7 or less - Fixes #7265. Signed-off-by: Ivan Rakov <ira...@apache.org> (cherry picked from commit e31b5300363e2c6a5f51374b8ba4bf6552fd2406) Signed-off-by: Ivan Rakov <ira...@apache.org> --- .../IgnitePKIndexesMigrationToUnwrapPkTest.java | 1 - .../persistence/MetaStorageCompatibilityTest.java | 348 +++++++++++++++++++++ .../IgniteCompatibilityBasicTestSuite.java | 6 +- .../cache/persistence/metastorage/MetaStorage.java | 46 ++- .../metastorage/IgniteMetaStorageBasicTest.java | 4 +- 5 files changed, 386 insertions(+), 19 deletions(-) diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/IgnitePKIndexesMigrationToUnwrapPkTest.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/IgnitePKIndexesMigrationToUnwrapPkTest.java index ed83ecd..3345d7b 100644 --- a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/IgnitePKIndexesMigrationToUnwrapPkTest.java +++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/IgnitePKIndexesMigrationToUnwrapPkTest.java @@ -1,4 +1,3 @@ - /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/MetaStorageCompatibilityTest.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/MetaStorageCompatibilityTest.java new file mode 100644 index 0000000..0933808 --- /dev/null +++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/MetaStorageCompatibilityTest.java @@ -0,0 +1,348 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.ignite.compatibility.persistence; + +import java.io.File; +import java.io.IOException; +import java.nio.file.OpenOption; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteException; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgnitionEx; +import org.apache.ignite.internal.processors.cache.GridCacheAbstractFullApiSelfTest; +import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIO; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory; +import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteInClosure; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.testframework.GridTestUtils; +import org.jetbrains.annotations.Nullable; +import org.junit.Test; + +/** + * Tests migration of metastorage. + */ +public class MetaStorageCompatibilityTest extends IgnitePersistenceCompatibilityAbstractTest { + /** Consistent id. */ + private static final String CONSISTENT_ID_1 = "node1"; + + /** Consistent id. */ + private static final String CONSISTENT_ID_2 = "node2"; + + /** Ignite version. */ + private static final String IGNITE_VERSION = "2.4.0"; + + /** Filename of index. */ + private static final String INDEX_BIN_FILE = "index.bin"; + + /** Filename of partition. */ + private static final String PART_FILE = "part-0.bin"; + + /** + * The zero partition hasn't full baseline information. The last changes are read from WAL. + * + * @throws Exception If failed. + */ + @Test + public void testMigrationWithoutFullBaselineIntoPartition() throws Exception { + try { + U.delete(new File(U.defaultWorkDirectory())); + + startGrid(1, IGNITE_VERSION, new ConfigurationClosure(CONSISTENT_ID_1), new ActivateAndForceCheckpointClosure()); + + startGrid(2, IGNITE_VERSION, new ConfigurationClosure(CONSISTENT_ID_2), new NewBaselineTopologyClosure()); + + stopAllGrids(); + + try (Ignite ig0 = IgnitionEx.start(prepareConfig(getConfiguration(), CONSISTENT_ID_1))) { + try (Ignite ig1 = IgnitionEx.start(prepareConfig(getConfiguration(), CONSISTENT_ID_2))) { + // No-op. + } + } + + assertFalse(metastorageFileExists(INDEX_BIN_FILE)); + assertFalse(metastorageFileExists(PART_FILE)); + + try (Ignite ig0 = IgnitionEx.start(prepareConfig(getConfiguration(), CONSISTENT_ID_1))) { + try (Ignite ig1 = IgnitionEx.start(prepareConfig(getConfiguration(), CONSISTENT_ID_2))) { + assertTrue(GridTestUtils.waitForCondition(() -> ig1.cluster().active(), 10_000)); + } + } + } + finally { + stopAllGrids(); + } + } + + /** + * Tests that BLT can be changed and persisted after metastorage migration. + */ + @Test + public void testMigrationToNewBaselineSetNewBaselineAfterMigration() throws Exception { + try { + U.delete(new File(U.defaultWorkDirectory())); + + startGrid(1, IGNITE_VERSION, new ConfigurationClosure(CONSISTENT_ID_1), new ActivateAndStopClosure()); + + stopAllGrids(); + + try (Ignite ig0 = IgnitionEx.start(prepareConfig(getConfiguration(), CONSISTENT_ID_1))) { + try (Ignite ig1 = IgnitionEx.start(prepareConfig(getConfiguration(), CONSISTENT_ID_2))) { + ig0.cluster().setBaselineTopology(ig1.cluster().topologyVersion()); + } + } + + assertFalse(metastorageFileExists(INDEX_BIN_FILE)); + assertFalse(metastorageFileExists(PART_FILE)); + + try (Ignite ig0 = IgnitionEx.start(prepareConfig(getConfiguration(), CONSISTENT_ID_1))) { + try (Ignite ig1 = IgnitionEx.start(prepareConfig(getConfiguration(), CONSISTENT_ID_2))) { + assertTrue(GridTestUtils.waitForCondition(() -> ig1.cluster().active(), 10_000)); + } + } + } + finally { + stopAllGrids(); + } + } + + /** + * + */ + @Test + public void testMigrationWithExceptionDuringTheProcess() throws Exception { + try { + U.delete(new File(U.defaultWorkDirectory())); + + startGrid(1, IGNITE_VERSION, new ConfigurationClosure(CONSISTENT_ID_1), new ActivateAndStopClosure()); + + stopAllGrids(); + + try (Ignite ig0 = IgnitionEx.start(prepareConfig(getConfiguration(), CONSISTENT_ID_1, PART_FILE))) { + ig0.getOrCreateCache("default-cache").put(1, 1); // trigger checkpoint on close() + } + + assertTrue(metastorageFileExists(INDEX_BIN_FILE)); + assertTrue(metastorageFileExists(PART_FILE)); + + try (Ignite ig0 = IgnitionEx.start(prepareConfig(getConfiguration(), CONSISTENT_ID_1, INDEX_BIN_FILE))) { + ig0.getOrCreateCache("default-cache").put(1, 1); // trigger checkpoint on close() + } + + assertTrue(metastorageFileExists(INDEX_BIN_FILE)); + assertFalse(metastorageFileExists(PART_FILE)); + + try (Ignite ig0 = IgnitionEx.start(prepareConfig(getConfiguration(), CONSISTENT_ID_1))) { + try (Ignite ig1 = IgnitionEx.start(prepareConfig(getConfiguration(), CONSISTENT_ID_2))) { + ig0.cluster().setBaselineTopology(ig1.cluster().topologyVersion()); + } + } + + try (Ignite ig0 = IgnitionEx.start(prepareConfig(getConfiguration(), CONSISTENT_ID_1))) { + try (Ignite ig1 = IgnitionEx.start(prepareConfig(getConfiguration(), CONSISTENT_ID_2))) { + assertTrue(GridTestUtils.waitForCondition(() -> ig1.cluster().active(), 10_000)); + } + } + } + finally { + stopAllGrids(); + } + } + + /** + * Checks that file exists. + * + * @param fileName File name. + */ + private boolean metastorageFileExists(String fileName) throws IgniteCheckedException { + return new File(U.defaultWorkDirectory() + "/db/" + U.maskForFileName(CONSISTENT_ID_1) + "/metastorage/" + fileName).exists(); + } + + /** + * Updates the given ignite configuration. + * + * @param cfg Ignite configuration to be updated. + * @param consistentId Consistent id. + * @return Updated configuration. + */ + private static IgniteConfiguration prepareConfig(IgniteConfiguration cfg, @Nullable String consistentId) { + return prepareConfig(cfg, consistentId, null); + } + + /** + * Updates the given ignite configuration. + * + * @param cfg Ignite configuration to be updated. + * @param consistentId Consistent id. + * @param failOnFileRmv Indicates that an exception should be trown when partition/index file is going to be removed. + * @return Updated configuration. + */ + private static IgniteConfiguration prepareConfig( + IgniteConfiguration cfg, + @Nullable String consistentId, + @Nullable String failOnFileRmv + ) { + cfg.setLocalHost("127.0.0.1"); + + TcpDiscoverySpi disco = new TcpDiscoverySpi(); + disco.setIpFinder(GridCacheAbstractFullApiSelfTest.LOCAL_IP_FINDER); + + cfg.setDiscoverySpi(disco); + + cfg.setPeerClassLoadingEnabled(false); + + DataStorageConfiguration memCfg = new DataStorageConfiguration() + .setDefaultDataRegionConfiguration( + new DataRegionConfiguration() + .setPersistenceEnabled(true) + .setInitialSize(10L * 1024 * 1024) + .setMaxSize(10L * 1024 * 1024)) + .setPageSize(4096); + + cfg.setDataStorageConfiguration(memCfg); + + if (consistentId != null) { + cfg.setIgniteInstanceName(consistentId); + cfg.setConsistentId(consistentId); + } + + if (failOnFileRmv != null) + cfg.getDataStorageConfiguration().setFileIOFactory(new FailingFileIOFactory(failOnFileRmv)); + + return cfg; + } + + /** */ + private static class ConfigurationClosure implements IgniteInClosure<IgniteConfiguration> { + /** Consistent id. */ + private final String consistentId; + + /** + * Creates a new instance of Configuration closure. + * + * @param consistentId Consistent id. + */ + public ConfigurationClosure(String consistentId) { + this.consistentId = consistentId; + } + + /** {@inheritDoc} */ + @Override public void apply(IgniteConfiguration cfg) { + prepareConfig(cfg, consistentId, null); + } + } + + /** + * Post-startup close that activate the grid and force checkpoint. + */ + private static class ActivateAndForceCheckpointClosure implements IgniteInClosure<Ignite> { + /** {@inheritDoc} */ + @Override public void apply(Ignite ignite) { + try { + ignite.active(true); + + ((IgniteEx)ignite).context().cache().context().database() + .forceCheckpoint("force test checkpoint").finishFuture().get(); + + ((GridCacheDatabaseSharedManager)(((IgniteEx)ignite).context().cache().context().database())) + .enableCheckpoints(false); + } + catch (IgniteCheckedException e) { + throw new IgniteException(e); + } + } + } + + /** + * Activates the cluster and stops it after that. + */ + private static class ActivateAndStopClosure implements IgniteInClosure<Ignite> { + /** {@inheritDoc} */ + @Override public void apply(Ignite ignite) { + ignite.active(true); + + ignite.close(); + } + } + + /** + * Disables checkpointer and sets a new baseline topology. + */ + private static class NewBaselineTopologyClosure implements IgniteInClosure<Ignite> { + /** {@inheritDoc} */ + @Override public void apply(Ignite ignite) { + ((GridCacheDatabaseSharedManager)(((IgniteEx)ignite).context().cache().context().database())) + .enableCheckpoints(false); + + long newTopVer = ignite.cluster().topologyVersion(); + + ignite.cluster().setBaselineTopology(newTopVer); + } + } + + /** + * Create File I/O which fails after second attempt to write to File + */ + private static class FailingFileIOFactory implements FileIOFactory { + /** Serial version uid. */ + private static final long serialVersionUID = 0L; + + /** Indicates that an exception should be trown when this file is going to be removed. */ + private final String failOnFileRmv; + + /** */ + private final FileIOFactory delegateFactory = new RandomAccessFileIOFactory(); + + /** + * Creates a new instance of IO factory. + */ + public FailingFileIOFactory() { + failOnFileRmv = null; + } + + /** + * Creates a new instance of IO factory. + * + * @param failOnFileRmv Indicates that an exception should be trown when the given file is removed. + */ + public FailingFileIOFactory(String failOnFileRmv) { + this.failOnFileRmv = failOnFileRmv; + } + + /** {@inheritDoc} */ + @Override public FileIO create(File file, OpenOption... modes) throws IOException { + final FileIO delegate = delegateFactory.create(file, modes); + + return new FileIODecorator(delegate) { + @Override public void clear() throws IOException { + if (failOnFileRmv != null && failOnFileRmv.equals(file.getName())) + throw new IOException("Test remove fail!"); + + super.clear(); + } + }; + } + } +} diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/IgniteCompatibilityBasicTestSuite.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/IgniteCompatibilityBasicTestSuite.java index 19ad466..e940ff4 100644 --- a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/IgniteCompatibilityBasicTestSuite.java +++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/IgniteCompatibilityBasicTestSuite.java @@ -18,6 +18,7 @@ package org.apache.ignite.compatibility.testsuites; import org.apache.ignite.compatibility.persistence.FoldersReuseCompatibilityTest; +import org.apache.ignite.compatibility.persistence.MetaStorageCompatibilityTest; import org.apache.ignite.compatibility.persistence.MigratingToWalV2SerializerWithCompactionTest; import org.apache.ignite.compatibility.persistence.PersistenceBasicCompatibilityTest; import org.junit.runner.RunWith; @@ -29,10 +30,9 @@ import org.junit.runners.Suite; @RunWith(Suite.class) @Suite.SuiteClasses({ PersistenceBasicCompatibilityTest.class, - FoldersReuseCompatibilityTest.class, - - MigratingToWalV2SerializerWithCompactionTest.class + MigratingToWalV2SerializerWithCompactionTest.class, + MetaStorageCompatibilityTest.class }) public class IgniteCompatibilityBasicTestSuite { } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java index 8db6d2c..f437126 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java @@ -44,12 +44,12 @@ import org.apache.ignite.internal.pagemem.FullPageId; import org.apache.ignite.internal.pagemem.PageIdAllocator; import org.apache.ignite.internal.pagemem.PageIdUtils; import org.apache.ignite.internal.pagemem.PageMemory; +import org.apache.ignite.internal.pagemem.store.PageStore; import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager; import org.apache.ignite.internal.pagemem.wal.WALPointer; import org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord; import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRecord; import org.apache.ignite.internal.processors.cache.CacheDiagnosticManager; -import org.apache.ignite.internal.processors.cache.CacheGroupContext; import org.apache.ignite.internal.processors.cache.GridCacheProcessor; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; import org.apache.ignite.internal.processors.cache.persistence.DataRegion; @@ -59,6 +59,7 @@ import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabase import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.cache.persistence.RootPage; import org.apache.ignite.internal.processors.cache.persistence.StorageException; +import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx; import org.apache.ignite.internal.processors.cache.persistence.partstorage.PartitionMetaStorageImpl; import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; @@ -177,14 +178,29 @@ public class MetaStorage implements DbCheckpointListener, ReadWriteMetastorage { gcProcessor.setTmpStorage(null); - // remove old partitions - CacheGroupContext cgc = cctx.cache().cacheGroup(METASTORAGE_CACHE_ID); + db.addCheckpointListener(new DbCheckpointListener() { + /** {@inheritDoc} */ + @Override public void onMarkCheckpointBegin(Context ctx) { + } - if (cgc != null) { - db.schedulePartitionDestroy(METASTORAGE_CACHE_ID, OLD_METASTORE_PARTITION); + /** {@inheritDoc} */ + @Override public void onCheckpointBegin(Context ctx) throws IgniteCheckedException { + assert cctx.pageStore() != null; - db.schedulePartitionDestroy(METASTORAGE_CACHE_ID, PageIdAllocator.INDEX_PARTITION); - } + int partTag = ((PageMemoryEx)dataRegion.pageMemory()).invalidate(METASTORAGE_CACHE_ID, OLD_METASTORE_PARTITION); + cctx.pageStore().onPartitionDestroyed(METASTORAGE_CACHE_ID, OLD_METASTORE_PARTITION, partTag); + + int idxTag = ((PageMemoryEx)dataRegion.pageMemory()).invalidate(METASTORAGE_CACHE_ID, PageIdAllocator.INDEX_PARTITION); + PageStore store = ((FilePageStoreManager)cctx.pageStore()).getStore(METASTORAGE_CACHE_ID, PageIdAllocator.INDEX_PARTITION); + store.truncate(idxTag); + + db.removeCheckpointListener(this); + } + + /** {@inheritDoc} */ + @Override public void beforeCheckpointBegin(Context ctx) { + } + }); } } } @@ -391,7 +407,7 @@ public class MetaStorage implements DbCheckpointListener, ReadWriteMetastorage { while (cur.next()) { MetastorageDataRow row = cur.get(); - res.add(new IgniteBiTuple<>(row.key(), row.value())); + res.add(new IgniteBiTuple<>(row.key(), marshaller.unmarshal(row.value(), getClass().getClassLoader()))); } return res; @@ -403,7 +419,15 @@ public class MetaStorage implements DbCheckpointListener, ReadWriteMetastorage { byte[] data = marshaller.marshal(val); - writeRaw(key, data); + final WALPointer ptr; + + synchronized (this) { + ptr = wal.log(new MetastoreDataRecord(key, data)); + + writeRaw(key, data); + } + + wal.flush(ptr, false); } /** {@inheritDoc} */ @@ -414,10 +438,6 @@ public class MetaStorage implements DbCheckpointListener, ReadWriteMetastorage { /** {@inheritDoc} */ @Override public void writeRaw(String key, byte[] data) throws IgniteCheckedException { if (!readOnly) { - WALPointer ptr = wal.log(new MetastoreDataRecord(key, data)); - - wal.flush(ptr, false); - synchronized (this) { MetastorageDataRow oldRow = tree.findOne(new MetastorageDataRow(key, null)); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java index 730d8e9..4361c4b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java @@ -171,7 +171,7 @@ public class IgniteMetaStorageBasicTest extends GridCommonAbstractTest { for (Iterator<IgniteBiTuple<String, byte[]>> it = generateTestData(size, from).iterator(); it.hasNext(); ) { IgniteBiTuple<String, byte[]> d = it.next(); - metaStorage.writeRaw(d.getKey(), d.getValue()); + metaStorage.write(d.getKey(), d.getValue()); res.put(d.getKey(), d.getValue()); } @@ -315,7 +315,7 @@ public class IgniteMetaStorageBasicTest extends GridCommonAbstractTest { try { for (Map.Entry<String, byte[]> v : testData.entrySet()) - metaStorage.writeRaw(v.getKey(), v.getValue()); + metaStorage.write(v.getKey(), v.getValue()); } finally { db.checkpointReadUnlock();