Mikhail Petrov created IGNITE-29131:
---------------------------------------
Summary: Idle verify running under load during node stop may
produce B+Tree is corrupted exception
Key: IGNITE-29131
URL: https://issues.apache.org/jira/browse/IGNITE-29131
Project: Ignite
Issue Type: Task
Reporter: Mikhail Petrov
Reproducer:
{code:java}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.ignite.internal.processors.cache.verify;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteDataStreamer;
import org.apache.ignite.cache.affinity.Affinity;
import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
import org.apache.ignite.cluster.ClusterState;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.DataRegionConfiguration;
import org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.failure.StopNodeOrHaltFailureHandler;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.junit.Test;
import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
/** */
public class IdleVerifyOrphanedTaskOnNodeStopTest extends
GridCommonAbstractTest {
/** {@inheritDoc} */
@Override protected IgniteConfiguration getConfiguration(String
igniteInstanceName) throws Exception {
return super.getConfiguration(igniteInstanceName)
.setDataStorageConfiguration(new DataStorageConfiguration()
.setDefaultDataRegionConfiguration(new DataRegionConfiguration()
.setPersistenceEnabled(true)))
.setCacheConfiguration(new CacheConfiguration<>(DEFAULT_CACHE_NAME)
.setAtomicityMode(TRANSACTIONAL)
.setBackups(2)
.setAffinity(new RendezvousAffinityFunction(false, 16)))
.setFailureHandler(new StopNodeOrHaltFailureHandler());
}
/** {@inheritDoc} */
@Override protected void beforeTest() throws Exception {
super.beforeTest();
cleanPersistenceDir();
}
/** {@inheritDoc} */
@Override protected void afterTest() throws Exception {
stopAllGrids();
cleanPersistenceDir();
super.afterTest();
}
/** */
@Test
public void testOrphanedIdleVerifyTaskOnNodeStop() throws Exception {
IgniteEx ignite = startGrid(0);
ignite.cluster().state(ClusterState.ACTIVE);
Affinity<Integer> aff = ignite.affinity(DEFAULT_CACHE_NAME);
try (IgniteDataStreamer<Integer, Integer> streamer =
ignite.dataStreamer(DEFAULT_CACHE_NAME)) {
for (int k = 0; k < 1_600_000; k++) {
if (aff.partition(k) != 0)
streamer.addData(k, k);
}
}
IgniteCache<Integer, Integer> cache = ignite.cache(DEFAULT_CACHE_NAME);
int keyUnderLoad = partitionKeys(cache, 0, 1, 0).get(0);
AtomicBoolean stop = new AtomicBoolean();
IgniteInternalFuture<?> loadFut = GridTestUtils.runAsync(() -> {
for (int i = 0; !stop.get(); i++)
cache.put(keyUnderLoad, i);
});
idleVerify(ignite, DEFAULT_CACHE_NAME);
stop.set(true);
loadFut.get();
stopGrid(0);
}
}
{code}
Exception with which the node fails:
{code:java}
class
org.apache.ignite.internal.processors.cache.persistence.tree.CorruptedTreeException:
B+Tree is corrupted [groupId=1544803905, pageIds=[1125929971613857],
msg=Runtime failure on cursor iteration]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.corruptedTreeException(BPlusTree.java:6052)
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$AbstractForwardCursor.nextPage(BPlusTree.java:5619)
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$ForwardCursor.next(BPlusTree.java:5788)
at
org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$3.onHasNext(IgniteCacheOffheapManagerImpl.java:687)
at
org.apache.ignite.internal.util.GridCloseableIteratorAdapter.hasNextX(GridCloseableIteratorAdapter.java:56)
at
org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.calculatePartitionHash(IdleVerifyUtility.java:311)
at
org.apache.ignite.internal.management.cache.VerifyBackupPartitionsTask$VerifyBackupPartitionsJob.lambda$calculatePartitionHashAsync$4(VerifyBackupPartitionsTask.java:515)
at
java.base/java.util.concurrent.FutureTask.run$$$capture(FutureTask.java:264)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
Caused by: java.lang.AssertionError
at
org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl.readUnlock(PageMemoryImpl.java:500)
at
org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler.readUnlock(PageHandler.java:222)
at
org.apache.ignite.internal.processors.cache.persistence.DataStructure.readUnlock(DataStructure.java:265)
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.access$6700(BPlusTree.java:214)
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$AbstractForwardCursor.nextPage(BPlusTree.java:5612)
... 10 more
{code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
