ibessonov commented on a change in pull request #9207: URL: https://github.com/apache/ignite/pull/9207#discussion_r676376058
########## File path: modules/core/src/main/java/org/apache/ignite/internal/cache/query/index/sorted/DurableBackgroundCleanupIndexTreeTaskV2.java ########## @@ -0,0 +1,539 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.cache.query.index.sorted; + +import java.io.Externalizable; +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.cache.query.index.sorted.inline.InlineIndexKeyType; +import org.apache.ignite.internal.cache.query.index.sorted.inline.InlineIndexTree; +import org.apache.ignite.internal.cache.query.index.sorted.keys.IndexKey; +import org.apache.ignite.internal.dto.IgniteDataTransferObject; +import org.apache.ignite.internal.pagemem.FullPageId; +import org.apache.ignite.internal.pagemem.PageMemory; +import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager; +import org.apache.ignite.internal.pagemem.wal.record.IndexRenameRootPageRecord; +import org.apache.ignite.internal.processors.cache.CacheGroupContext; +import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; +import org.apache.ignite.internal.processors.cache.persistence.RootPage; +import org.apache.ignite.internal.processors.cache.persistence.metastorage.pendingtask.DurableBackgroundTask; +import org.apache.ignite.internal.processors.cache.persistence.metastorage.pendingtask.DurableBackgroundTaskResult; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIoResolver; +import org.apache.ignite.internal.util.future.GridFinishedFuture; +import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.CU; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.internal.util.worker.GridWorker; +import org.apache.ignite.thread.IgniteThread; +import org.jetbrains.annotations.Nullable; + +import static java.util.Collections.emptyList; +import static java.util.Collections.singleton; + +/** + * Task for background cleaning of index trees. + */ +public class DurableBackgroundCleanupIndexTreeTaskV2 extends IgniteDataTransferObject implements + DurableBackgroundTask<Long> { + /** Serial version uid. */ + private static final long serialVersionUID = 0L; + + /** Index tree index factory. */ + public static InlineIndexTreeFactory IDX_TREE_FACTORY = new InlineIndexTreeFactory(); + + /** Logger. */ + @Nullable private transient volatile IgniteLogger log; + + /** Unique id. */ + private String uid; + + /** Cache group name. */ + @Nullable private String grpName; + + /** Cache name. */ + private String cacheName; + + /** Index name. */ + private String idxName; + + /** Old name of underlying index tree name. */ + private String oldTreeName; + + /** New name of underlying index tree name. */ + private String newTreeName; + + /** Number of segments. */ + private int segments; + + /** Need to rename index root pages. */ + private transient volatile boolean needToRen; + + /** Index root pages. Mapping: segment number -> index root page. */ + @Nullable private transient volatile Map<Integer, RootPage> rootPages; + + /** Worker cleaning index trees. */ + @Nullable private transient volatile GridWorker worker; + + /** Total number of pages recycled from index trees. */ + private final AtomicLong pageCnt = new AtomicLong(); + + /** + * Constructor. + * + * @param grpName Cache group name. + * @param cacheName Cache name. + * @param idxName Index name. + * @param oldTreeName Old name of underlying index tree name. + * @param newTreeName New name of underlying index tree name. + * @param segments Number of segments. + * @param trees Index trees. + */ + public DurableBackgroundCleanupIndexTreeTaskV2( + @Nullable String grpName, + String cacheName, + String idxName, + String oldTreeName, + String newTreeName, + int segments, + @Nullable InlineIndexTree[] trees + ) { + uid = UUID.randomUUID().toString(); + this.grpName = grpName; + this.cacheName = cacheName; + this.idxName = idxName; + this.oldTreeName = oldTreeName; + this.newTreeName = newTreeName; + this.segments = segments; + + if (trees != null) { + assert trees.length == segments : + "Invalid number of index trees [trees=" + trees.length + ", segments=" + segments + ']'; + + Map<Integer, RootPage> rootPages0 = new ConcurrentHashMap<>(); + + for (int i = 0; i < trees.length; i++) { + InlineIndexTree tree = trees[i]; + + assert tree != null; + + tree.close(); + + rootPages0.put(i, rootPage(tree)); + } + + this.rootPages = rootPages0; Review comment: Interesting patters, why not just use field instead of temporary variable? ########## File path: modules/core/src/main/java/org/apache/ignite/internal/processors/localtask/DurableBackgroundTasksProcessor.java ########## @@ -111,9 +111,22 @@ public DurableBackgroundTasksProcessor(GridKernalContext ctx) { metaStorage.iterate( TASK_PREFIX, (k, v) -> { - DurableBackgroundTask t = (DurableBackgroundTask)v; + DurableBackgroundTask t = ((DurableBackgroundTask<?>)v); + DurableBackgroundTask converted = t.convertAfterRestoreIfNeeded(); - tasks.put(t.name(), new DurableBackgroundTaskState(t, null, true)); + if (t != converted) { + GridFutureAdapter<?> outFut = new GridFutureAdapter<>(); + outFut.onDone(); + + DurableBackgroundTaskState<?> state = new DurableBackgroundTaskState<>(t, outFut, true); + state.state(COMPLETED); Review comment: Will this action lead to tasks deletion from metastorage on the next checkpoint even if v2 is not yet completed? ########## File path: modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DropIndexTest.java ########## @@ -0,0 +1,563 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.index; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ThreadLocalRandom; +import java.util.stream.IntStream; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.cache.query.SqlFieldsQuery; +import org.apache.ignite.client.Person; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.cache.query.index.Index; +import org.apache.ignite.internal.cache.query.index.sorted.DurableBackgroundCleanupIndexTreeTaskV2; +import org.apache.ignite.internal.cache.query.index.sorted.DurableBackgroundCleanupIndexTreeTaskV2.InlineIndexTreeFactory; +import org.apache.ignite.internal.cache.query.index.sorted.SortedIndexDefinition; +import org.apache.ignite.internal.cache.query.index.sorted.inline.InlineIndexTree; +import org.apache.ignite.internal.pagemem.FullPageId; +import org.apache.ignite.internal.processors.cache.CacheGroupContext; +import org.apache.ignite.internal.processors.cache.GridCacheContext; +import org.apache.ignite.internal.processors.cache.persistence.RootPage; +import org.apache.ignite.internal.processors.cache.persistence.metastorage.pendingtask.DurableBackgroundTaskResult; +import org.apache.ignite.internal.processors.localtask.DurableBackgroundTaskState; +import org.apache.ignite.internal.util.function.ThrowableFunction; +import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.testframework.junits.WithSystemProperty; +import org.jetbrains.annotations.Nullable; +import org.junit.Test; + +import static java.util.stream.Collectors.joining; +import static org.apache.ignite.IgniteSystemProperties.IGNITE_MAX_INDEX_PAYLOAD_SIZE; +import static org.apache.ignite.cluster.ClusterState.ACTIVE; +import static org.apache.ignite.cluster.ClusterState.INACTIVE; +import static org.apache.ignite.internal.cache.query.index.sorted.DurableBackgroundCleanupIndexTreeTaskV2.IDX_TREE_FACTORY; +import static org.apache.ignite.internal.cache.query.index.sorted.DurableBackgroundCleanupIndexTreeTaskV2.destroyIndexTrees; +import static org.apache.ignite.internal.cache.query.index.sorted.DurableBackgroundCleanupIndexTreeTaskV2.existMetaPage; +import static org.apache.ignite.internal.cache.query.index.sorted.DurableBackgroundCleanupIndexTreeTaskV2.findIndexRootPages; +import static org.apache.ignite.internal.cache.query.index.sorted.DurableBackgroundCleanupIndexTreeTaskV2.rootPage; +import static org.apache.ignite.testframework.GridTestUtils.cacheContext; +import static org.apache.ignite.testframework.GridTestUtils.getFieldValue; +import static org.apache.ignite.testframework.GridTestUtils.runAsync; + +/** + * Class for testing index drop. + */ +@WithSystemProperty(key = IGNITE_MAX_INDEX_PAYLOAD_SIZE, value = "1000000") +public class DropIndexTest extends AbstractRebuildIndexTest { + /** Original {@link DurableBackgroundCleanupIndexTreeTaskV2#IDX_TREE_FACTORY}. */ + private InlineIndexTreeFactory originalTaskIdxTreeFactory; + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + originalTaskIdxTreeFactory = IDX_TREE_FACTORY; + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + IDX_TREE_FACTORY = originalTaskIdxTreeFactory; + } + + /** {@inheritDoc} */ + @Override protected void populate(IgniteCache<Integer, Person> cache, int cnt) { + String prefix = IntStream.range(0, 1_000).mapToObj(i -> "name").collect(joining("_")) + "_"; + + for (int i = 0; i < cnt; i++) + cache.put(i, new Person(i, prefix + i)); + } + + /** + * Checking {@link DurableBackgroundCleanupIndexTreeTaskV2#destroyIndexTrees}. + * + * @throws Exception If failed. + */ + @Test + public void testDestroyIndexTrees() throws Exception { + checkDestroyIndexTrees(true, 3); + } + + /** + * Check that the {@link DurableBackgroundCleanupIndexTreeTaskV2} will not + * be executed if the cache group and root pages are not found. + * + * @throws Exception If failed. + */ + @Test + public void testTaskNotExecuteIfAbsentCacheGroupOrRootPages() throws Exception { + IgniteEx n = startGrid(0); + + String fake = IntStream.generate(() -> ThreadLocalRandom.current().nextInt()) Review comment: Wow, why does it have to be so complicated? ########## File path: modules/core/src/main/java/org/apache/ignite/internal/cache/query/index/sorted/DurableBackgroundCleanupIndexTreeTaskV2.java ########## @@ -0,0 +1,539 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.cache.query.index.sorted; + +import java.io.Externalizable; +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.cache.query.index.sorted.inline.InlineIndexKeyType; +import org.apache.ignite.internal.cache.query.index.sorted.inline.InlineIndexTree; +import org.apache.ignite.internal.cache.query.index.sorted.keys.IndexKey; +import org.apache.ignite.internal.dto.IgniteDataTransferObject; +import org.apache.ignite.internal.pagemem.FullPageId; +import org.apache.ignite.internal.pagemem.PageMemory; +import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager; +import org.apache.ignite.internal.pagemem.wal.record.IndexRenameRootPageRecord; +import org.apache.ignite.internal.processors.cache.CacheGroupContext; +import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; +import org.apache.ignite.internal.processors.cache.persistence.RootPage; +import org.apache.ignite.internal.processors.cache.persistence.metastorage.pendingtask.DurableBackgroundTask; +import org.apache.ignite.internal.processors.cache.persistence.metastorage.pendingtask.DurableBackgroundTaskResult; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIoResolver; +import org.apache.ignite.internal.util.future.GridFinishedFuture; +import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.CU; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.internal.util.worker.GridWorker; +import org.apache.ignite.thread.IgniteThread; +import org.jetbrains.annotations.Nullable; + +import static java.util.Collections.emptyList; +import static java.util.Collections.singleton; + +/** + * Task for background cleaning of index trees. + */ +public class DurableBackgroundCleanupIndexTreeTaskV2 extends IgniteDataTransferObject implements + DurableBackgroundTask<Long> { + /** Serial version uid. */ + private static final long serialVersionUID = 0L; + + /** Index tree index factory. */ + public static InlineIndexTreeFactory IDX_TREE_FACTORY = new InlineIndexTreeFactory(); + + /** Logger. */ + @Nullable private transient volatile IgniteLogger log; + + /** Unique id. */ + private String uid; + + /** Cache group name. */ + @Nullable private String grpName; + + /** Cache name. */ + private String cacheName; + + /** Index name. */ + private String idxName; + + /** Old name of underlying index tree name. */ + private String oldTreeName; + + /** New name of underlying index tree name. */ + private String newTreeName; + + /** Number of segments. */ + private int segments; + + /** Need to rename index root pages. */ + private transient volatile boolean needToRen; + + /** Index root pages. Mapping: segment number -> index root page. */ + @Nullable private transient volatile Map<Integer, RootPage> rootPages; + + /** Worker cleaning index trees. */ + @Nullable private transient volatile GridWorker worker; + + /** Total number of pages recycled from index trees. */ + private final AtomicLong pageCnt = new AtomicLong(); + + /** + * Constructor. + * + * @param grpName Cache group name. + * @param cacheName Cache name. + * @param idxName Index name. + * @param oldTreeName Old name of underlying index tree name. + * @param newTreeName New name of underlying index tree name. + * @param segments Number of segments. + * @param trees Index trees. + */ + public DurableBackgroundCleanupIndexTreeTaskV2( + @Nullable String grpName, + String cacheName, + String idxName, + String oldTreeName, + String newTreeName, + int segments, + @Nullable InlineIndexTree[] trees + ) { + uid = UUID.randomUUID().toString(); + this.grpName = grpName; + this.cacheName = cacheName; + this.idxName = idxName; + this.oldTreeName = oldTreeName; + this.newTreeName = newTreeName; + this.segments = segments; + + if (trees != null) { + assert trees.length == segments : + "Invalid number of index trees [trees=" + trees.length + ", segments=" + segments + ']'; + + Map<Integer, RootPage> rootPages0 = new ConcurrentHashMap<>(); + + for (int i = 0; i < trees.length; i++) { + InlineIndexTree tree = trees[i]; + + assert tree != null; + + tree.close(); + + rootPages0.put(i, rootPage(tree)); + } + + this.rootPages = rootPages0; + } + + needToRen = true; + } + + /** + * Default constructor for {@link Externalizable}. + */ + public DurableBackgroundCleanupIndexTreeTaskV2() { + // No-op. + } + + /** {@inheritDoc} */ + @Override protected void writeExternalData(ObjectOutput out) throws IOException { + U.writeLongString(out, uid); + U.writeLongString(out, grpName); + U.writeLongString(out, cacheName); + U.writeLongString(out, idxName); + U.writeLongString(out, oldTreeName); + U.writeLongString(out, newTreeName); + out.writeInt(segments); + } + + /** {@inheritDoc} */ + @Override protected void readExternalData( + byte protoVer, + ObjectInput in + ) throws IOException, ClassNotFoundException { + uid = U.readLongString(in); + grpName = U.readLongString(in); + cacheName = U.readLongString(in); + idxName = U.readLongString(in); + oldTreeName = U.readLongString(in); + newTreeName = U.readLongString(in); + segments = in.readInt(); + } + + /** {@inheritDoc} */ + @Override public String name() { + return "drop-sql-index-" + cacheName + "-" + idxName + "-" + uid; + } + + /** {@inheritDoc} */ + @Override public void cancel() { + rootPages = null; + + GridWorker w = worker; + + if (w != null) { + worker = null; + + U.awaitForWorkersStop(singleton(w), true, log); + } + } + + /** {@inheritDoc} */ + @Override public void onDeactivationCluster() { + rootPages = null; + } + + /** {@inheritDoc} */ + @Override public IgniteInternalFuture<DurableBackgroundTaskResult<Long>> executeAsync(GridKernalContext ctx) { + assert worker == null; + + log = ctx.log(DurableBackgroundCleanupIndexTreeTaskV2.class); + + IgniteInternalFuture<DurableBackgroundTaskResult<Long>> outFut; + + CacheGroupContext grpCtx = ctx.cache().cacheGroup(CU.cacheGroupId(cacheName, grpName)); + + if (grpCtx != null) { + try { + // Renaming should be done once when adding (and immediately launched) a task at the time of drop the index. + // To avoid problems due to node crash between renaming and adding a task. + if (needToRen) { + // If the node falls before renaming, then the index was definitely not dropped. + // If the node crashes after renaming, the task will delete the old index trees, + // and the node will rebuild this index when the node starts. + renameIndexRootPages(grpCtx, cacheName, oldTreeName, newTreeName, segments); + + // After restoring from MetaStorage, it will also be {@code false}. + needToRen = false; + } + + @Nullable Map<Integer, RootPage> rootPages0 = this.rootPages; Review comment: Are we protecting ourselves from "cancel" or deactivation here? I don't get why this code is written like this. Can we add task to the worker after "cancel" is invoked? ########## File path: modules/core/src/main/java/org/apache/ignite/internal/cache/query/index/sorted/DurableBackgroundCleanupIndexTreeTaskV2.java ########## @@ -0,0 +1,539 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.cache.query.index.sorted; + +import java.io.Externalizable; +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.cache.query.index.sorted.inline.InlineIndexKeyType; +import org.apache.ignite.internal.cache.query.index.sorted.inline.InlineIndexTree; +import org.apache.ignite.internal.cache.query.index.sorted.keys.IndexKey; +import org.apache.ignite.internal.dto.IgniteDataTransferObject; +import org.apache.ignite.internal.pagemem.FullPageId; +import org.apache.ignite.internal.pagemem.PageMemory; +import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager; +import org.apache.ignite.internal.pagemem.wal.record.IndexRenameRootPageRecord; +import org.apache.ignite.internal.processors.cache.CacheGroupContext; +import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; +import org.apache.ignite.internal.processors.cache.persistence.RootPage; +import org.apache.ignite.internal.processors.cache.persistence.metastorage.pendingtask.DurableBackgroundTask; +import org.apache.ignite.internal.processors.cache.persistence.metastorage.pendingtask.DurableBackgroundTaskResult; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIoResolver; +import org.apache.ignite.internal.util.future.GridFinishedFuture; +import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.CU; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.internal.util.worker.GridWorker; +import org.apache.ignite.thread.IgniteThread; +import org.jetbrains.annotations.Nullable; + +import static java.util.Collections.emptyList; +import static java.util.Collections.singleton; + +/** + * Task for background cleaning of index trees. + */ +public class DurableBackgroundCleanupIndexTreeTaskV2 extends IgniteDataTransferObject implements + DurableBackgroundTask<Long> { + /** Serial version uid. */ + private static final long serialVersionUID = 0L; + + /** Index tree index factory. */ + public static InlineIndexTreeFactory IDX_TREE_FACTORY = new InlineIndexTreeFactory(); Review comment: Why not "final"? ########## File path: modules/core/src/main/java/org/apache/ignite/internal/processors/localtask/DurableBackgroundTasksProcessor.java ########## @@ -111,9 +111,22 @@ public DurableBackgroundTasksProcessor(GridKernalContext ctx) { metaStorage.iterate( TASK_PREFIX, (k, v) -> { - DurableBackgroundTask t = (DurableBackgroundTask)v; + DurableBackgroundTask t = ((DurableBackgroundTask<?>)v); + DurableBackgroundTask converted = t.convertAfterRestoreIfNeeded(); - tasks.put(t.name(), new DurableBackgroundTaskState(t, null, true)); + if (t != converted) { + GridFutureAdapter<?> outFut = new GridFutureAdapter<>(); + outFut.onDone(); + + DurableBackgroundTaskState<?> state = new DurableBackgroundTaskState<>(t, outFut, true); + state.state(COMPLETED); + + tasks.put(t.name(), state); + + t = converted; + } + + tasks.put(t.name(), new DurableBackgroundTaskState<>(t, new GridFutureAdapter<>(), true)); Review comment: I don't really understand, you're doing "put" in "if" branch and immediately doing another "put" outside of it. Should there be "else" here somewhere? ########## File path: modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DropIndexTest.java ########## @@ -0,0 +1,563 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.index; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ThreadLocalRandom; +import java.util.stream.IntStream; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.cache.query.SqlFieldsQuery; +import org.apache.ignite.client.Person; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.cache.query.index.Index; +import org.apache.ignite.internal.cache.query.index.sorted.DurableBackgroundCleanupIndexTreeTaskV2; +import org.apache.ignite.internal.cache.query.index.sorted.DurableBackgroundCleanupIndexTreeTaskV2.InlineIndexTreeFactory; +import org.apache.ignite.internal.cache.query.index.sorted.SortedIndexDefinition; +import org.apache.ignite.internal.cache.query.index.sorted.inline.InlineIndexTree; +import org.apache.ignite.internal.pagemem.FullPageId; +import org.apache.ignite.internal.processors.cache.CacheGroupContext; +import org.apache.ignite.internal.processors.cache.GridCacheContext; +import org.apache.ignite.internal.processors.cache.persistence.RootPage; +import org.apache.ignite.internal.processors.cache.persistence.metastorage.pendingtask.DurableBackgroundTaskResult; +import org.apache.ignite.internal.processors.localtask.DurableBackgroundTaskState; +import org.apache.ignite.internal.util.function.ThrowableFunction; +import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.testframework.junits.WithSystemProperty; +import org.jetbrains.annotations.Nullable; +import org.junit.Test; + +import static java.util.stream.Collectors.joining; +import static org.apache.ignite.IgniteSystemProperties.IGNITE_MAX_INDEX_PAYLOAD_SIZE; +import static org.apache.ignite.cluster.ClusterState.ACTIVE; +import static org.apache.ignite.cluster.ClusterState.INACTIVE; +import static org.apache.ignite.internal.cache.query.index.sorted.DurableBackgroundCleanupIndexTreeTaskV2.IDX_TREE_FACTORY; +import static org.apache.ignite.internal.cache.query.index.sorted.DurableBackgroundCleanupIndexTreeTaskV2.destroyIndexTrees; +import static org.apache.ignite.internal.cache.query.index.sorted.DurableBackgroundCleanupIndexTreeTaskV2.existMetaPage; +import static org.apache.ignite.internal.cache.query.index.sorted.DurableBackgroundCleanupIndexTreeTaskV2.findIndexRootPages; +import static org.apache.ignite.internal.cache.query.index.sorted.DurableBackgroundCleanupIndexTreeTaskV2.rootPage; +import static org.apache.ignite.testframework.GridTestUtils.cacheContext; +import static org.apache.ignite.testframework.GridTestUtils.getFieldValue; +import static org.apache.ignite.testframework.GridTestUtils.runAsync; + +/** + * Class for testing index drop. + */ +@WithSystemProperty(key = IGNITE_MAX_INDEX_PAYLOAD_SIZE, value = "1000000") +public class DropIndexTest extends AbstractRebuildIndexTest { + /** Original {@link DurableBackgroundCleanupIndexTreeTaskV2#IDX_TREE_FACTORY}. */ + private InlineIndexTreeFactory originalTaskIdxTreeFactory; + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + originalTaskIdxTreeFactory = IDX_TREE_FACTORY; Review comment: Oh, I see now why it's not final. Naming notation implies that it should be a constant. Can you reflect this moment in the comment on IDX_TREE_FACTORY? ########## File path: modules/core/src/main/java/org/apache/ignite/internal/cache/query/index/sorted/DurableBackgroundCleanupIndexTreeTaskV2.java ########## @@ -0,0 +1,539 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.cache.query.index.sorted; + +import java.io.Externalizable; +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.cache.query.index.sorted.inline.InlineIndexKeyType; +import org.apache.ignite.internal.cache.query.index.sorted.inline.InlineIndexTree; +import org.apache.ignite.internal.cache.query.index.sorted.keys.IndexKey; +import org.apache.ignite.internal.dto.IgniteDataTransferObject; +import org.apache.ignite.internal.pagemem.FullPageId; +import org.apache.ignite.internal.pagemem.PageMemory; +import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager; +import org.apache.ignite.internal.pagemem.wal.record.IndexRenameRootPageRecord; +import org.apache.ignite.internal.processors.cache.CacheGroupContext; +import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; +import org.apache.ignite.internal.processors.cache.persistence.RootPage; +import org.apache.ignite.internal.processors.cache.persistence.metastorage.pendingtask.DurableBackgroundTask; +import org.apache.ignite.internal.processors.cache.persistence.metastorage.pendingtask.DurableBackgroundTaskResult; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIoResolver; +import org.apache.ignite.internal.util.future.GridFinishedFuture; +import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.CU; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.internal.util.worker.GridWorker; +import org.apache.ignite.thread.IgniteThread; +import org.jetbrains.annotations.Nullable; + +import static java.util.Collections.emptyList; +import static java.util.Collections.singleton; + +/** + * Task for background cleaning of index trees. + */ +public class DurableBackgroundCleanupIndexTreeTaskV2 extends IgniteDataTransferObject implements + DurableBackgroundTask<Long> { + /** Serial version uid. */ + private static final long serialVersionUID = 0L; + + /** Index tree index factory. */ + public static InlineIndexTreeFactory IDX_TREE_FACTORY = new InlineIndexTreeFactory(); + + /** Logger. */ + @Nullable private transient volatile IgniteLogger log; + + /** Unique id. */ + private String uid; + + /** Cache group name. */ + @Nullable private String grpName; + + /** Cache name. */ + private String cacheName; + + /** Index name. */ + private String idxName; + + /** Old name of underlying index tree name. */ + private String oldTreeName; + + /** New name of underlying index tree name. */ + private String newTreeName; + + /** Number of segments. */ + private int segments; + + /** Need to rename index root pages. */ + private transient volatile boolean needToRen; + + /** Index root pages. Mapping: segment number -> index root page. */ + @Nullable private transient volatile Map<Integer, RootPage> rootPages; + + /** Worker cleaning index trees. */ + @Nullable private transient volatile GridWorker worker; + + /** Total number of pages recycled from index trees. */ + private final AtomicLong pageCnt = new AtomicLong(); Review comment: Should this one be transient? Given that the class is IgniteDTO, why do we even need to mark certain fields? ########## File path: modules/core/src/main/java/org/apache/ignite/internal/cache/query/index/sorted/inline/InlineIndexImpl.java ########## @@ -422,61 +421,37 @@ private static boolean isExpired(IndexRow row) { private final AtomicBoolean destroyed = new AtomicBoolean(); /** {@inheritDoc} */ - @Override public void destroy(boolean softDelete) { + @Override public void destroy(boolean softDel) { // Already destroyed. if (!destroyed.compareAndSet(false, true)) return; - try { - if (cctx.affinityNode() && !softDelete) { - List<Long> rootPages = new ArrayList<>(segments.length); - List<InlineIndexTree> trees = new ArrayList<>(segments.length); - - cctx.shared().database().checkpointReadLock(); - - try { - for (int i = 0; i < segments.length; i++) { - InlineIndexTree tree = segments[i]; - - // Just mark it as destroyed. Actual destroy later in background task. - tree.markDestroyed(); + if (cctx.affinityNode() && !softDel) { + for (InlineIndexTree segment : segments) + segment.markDestroyed(); - rootPages.add(tree.getMetaPageId()); - trees.add(tree); - - dropMetaPage(i); - } - } - finally { - cctx.shared().database().checkpointReadUnlock(); - } - - cctx.kernalContext().metric().remove(stats.metricRegistryName()); + cctx.kernalContext().metric().remove(stats.metricRegistryName()); + if (cctx.group().persistenceEnabled() || + cctx.shared().kernalContext().state().clusterState().state() != INACTIVE) { Review comment: I'm not sure I understand this condition. Why don't we create the task on inactive cluster? ########## File path: modules/core/src/test/java/org/apache/ignite/internal/processors/localtask/DurableBackgroundTasksProcessorSelfTest.java ########## @@ -279,6 +281,44 @@ public void testDontDeleteTaskIfItsRestart() throws Exception { assertNull(metaStorageOperation(n, ms -> ms.read(metaStorageKey(simpleTask1)))); } + /** + * Checking the correctness of using the {@link DurableBackgroundTask#convertAfterRestoreIfNeeded}. + * + * @throws Exception If failed. + */ + @Test + public void testConvertAfterRestoreIfNeeded() throws Exception { + IgniteEx n = startGrid(0); + n.cluster().state(ACTIVE); + + String taskName0 = "test-task0"; + String taskName1 = "test-task1"; + + ConvertibleTask t0 = new ConvertibleTask(taskName0); + SimpleTask t1 = new SimpleTask(taskName1); + + SimpleTask t2 = (SimpleTask)t0.convertAfterRestoreIfNeeded(); + assertEquals("converted-task-" + taskName0, t2.name()); + + assertNotNull(n.context().durableBackgroundTask().executeAsync(t0, true)); + assertNotNull(n.context().durableBackgroundTask().executeAsync(t1, true)); + + assertEquals(2, tasks(n).size()); + + checkStateAndMetaStorage(n, t0, STARTED, true, false); + checkStateAndMetaStorage(n, t1, STARTED, true, false); + + stopGrid(0); + + n = startGrid(0); + + assertEquals(3, tasks(n).size()); + + checkStateAndMetaStorage(n, t0, COMPLETED, true, true); + checkStateAndMetaStorage(n, t1, INIT, true, false); + checkStateAndMetaStorage(n, t1, INIT, true, false); Review comment: I see this line twice. Did you mean "t2"? ########## File path: modules/core/src/main/java/org/apache/ignite/internal/cache/query/index/sorted/DurableBackgroundCleanupIndexTreeTaskV2.java ########## @@ -0,0 +1,539 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.cache.query.index.sorted; + +import java.io.Externalizable; +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.cache.query.index.sorted.inline.InlineIndexKeyType; +import org.apache.ignite.internal.cache.query.index.sorted.inline.InlineIndexTree; +import org.apache.ignite.internal.cache.query.index.sorted.keys.IndexKey; +import org.apache.ignite.internal.dto.IgniteDataTransferObject; +import org.apache.ignite.internal.pagemem.FullPageId; +import org.apache.ignite.internal.pagemem.PageMemory; +import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager; +import org.apache.ignite.internal.pagemem.wal.record.IndexRenameRootPageRecord; +import org.apache.ignite.internal.processors.cache.CacheGroupContext; +import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; +import org.apache.ignite.internal.processors.cache.persistence.RootPage; +import org.apache.ignite.internal.processors.cache.persistence.metastorage.pendingtask.DurableBackgroundTask; +import org.apache.ignite.internal.processors.cache.persistence.metastorage.pendingtask.DurableBackgroundTaskResult; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIoResolver; +import org.apache.ignite.internal.util.future.GridFinishedFuture; +import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.CU; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.internal.util.worker.GridWorker; +import org.apache.ignite.thread.IgniteThread; +import org.jetbrains.annotations.Nullable; + +import static java.util.Collections.emptyList; +import static java.util.Collections.singleton; + +/** + * Task for background cleaning of index trees. + */ +public class DurableBackgroundCleanupIndexTreeTaskV2 extends IgniteDataTransferObject implements + DurableBackgroundTask<Long> { + /** Serial version uid. */ + private static final long serialVersionUID = 0L; + + /** Index tree index factory. */ + public static InlineIndexTreeFactory IDX_TREE_FACTORY = new InlineIndexTreeFactory(); + + /** Logger. */ + @Nullable private transient volatile IgniteLogger log; + + /** Unique id. */ + private String uid; + + /** Cache group name. */ + @Nullable private String grpName; + + /** Cache name. */ + private String cacheName; + + /** Index name. */ + private String idxName; + + /** Old name of underlying index tree name. */ + private String oldTreeName; + + /** New name of underlying index tree name. */ + private String newTreeName; + + /** Number of segments. */ + private int segments; + + /** Need to rename index root pages. */ + private transient volatile boolean needToRen; + + /** Index root pages. Mapping: segment number -> index root page. */ + @Nullable private transient volatile Map<Integer, RootPage> rootPages; + + /** Worker cleaning index trees. */ + @Nullable private transient volatile GridWorker worker; + + /** Total number of pages recycled from index trees. */ + private final AtomicLong pageCnt = new AtomicLong(); + + /** + * Constructor. + * + * @param grpName Cache group name. + * @param cacheName Cache name. + * @param idxName Index name. + * @param oldTreeName Old name of underlying index tree name. + * @param newTreeName New name of underlying index tree name. + * @param segments Number of segments. + * @param trees Index trees. + */ + public DurableBackgroundCleanupIndexTreeTaskV2( + @Nullable String grpName, + String cacheName, + String idxName, + String oldTreeName, + String newTreeName, + int segments, + @Nullable InlineIndexTree[] trees + ) { + uid = UUID.randomUUID().toString(); + this.grpName = grpName; + this.cacheName = cacheName; + this.idxName = idxName; + this.oldTreeName = oldTreeName; + this.newTreeName = newTreeName; + this.segments = segments; + + if (trees != null) { + assert trees.length == segments : + "Invalid number of index trees [trees=" + trees.length + ", segments=" + segments + ']'; + + Map<Integer, RootPage> rootPages0 = new ConcurrentHashMap<>(); + + for (int i = 0; i < trees.length; i++) { + InlineIndexTree tree = trees[i]; + + assert tree != null; + + tree.close(); Review comment: Is this appropriate? Why does this constructor closes resources that don't belong to the object? ########## File path: modules/core/src/main/java/org/apache/ignite/internal/cache/query/index/sorted/DurableBackgroundCleanupIndexTreeTaskV2.java ########## @@ -0,0 +1,539 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.cache.query.index.sorted; + +import java.io.Externalizable; +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.cache.query.index.sorted.inline.InlineIndexKeyType; +import org.apache.ignite.internal.cache.query.index.sorted.inline.InlineIndexTree; +import org.apache.ignite.internal.cache.query.index.sorted.keys.IndexKey; +import org.apache.ignite.internal.dto.IgniteDataTransferObject; +import org.apache.ignite.internal.pagemem.FullPageId; +import org.apache.ignite.internal.pagemem.PageMemory; +import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager; +import org.apache.ignite.internal.pagemem.wal.record.IndexRenameRootPageRecord; +import org.apache.ignite.internal.processors.cache.CacheGroupContext; +import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; +import org.apache.ignite.internal.processors.cache.persistence.RootPage; +import org.apache.ignite.internal.processors.cache.persistence.metastorage.pendingtask.DurableBackgroundTask; +import org.apache.ignite.internal.processors.cache.persistence.metastorage.pendingtask.DurableBackgroundTaskResult; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIoResolver; +import org.apache.ignite.internal.util.future.GridFinishedFuture; +import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.CU; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.internal.util.worker.GridWorker; +import org.apache.ignite.thread.IgniteThread; +import org.jetbrains.annotations.Nullable; + +import static java.util.Collections.emptyList; +import static java.util.Collections.singleton; + +/** + * Task for background cleaning of index trees. + */ +public class DurableBackgroundCleanupIndexTreeTaskV2 extends IgniteDataTransferObject implements + DurableBackgroundTask<Long> { + /** Serial version uid. */ + private static final long serialVersionUID = 0L; + + /** Index tree index factory. */ + public static InlineIndexTreeFactory IDX_TREE_FACTORY = new InlineIndexTreeFactory(); + + /** Logger. */ + @Nullable private transient volatile IgniteLogger log; + + /** Unique id. */ + private String uid; + + /** Cache group name. */ + @Nullable private String grpName; + + /** Cache name. */ + private String cacheName; + + /** Index name. */ + private String idxName; + + /** Old name of underlying index tree name. */ + private String oldTreeName; + + /** New name of underlying index tree name. */ + private String newTreeName; + + /** Number of segments. */ + private int segments; + + /** Need to rename index root pages. */ + private transient volatile boolean needToRen; + + /** Index root pages. Mapping: segment number -> index root page. */ + @Nullable private transient volatile Map<Integer, RootPage> rootPages; + + /** Worker cleaning index trees. */ + @Nullable private transient volatile GridWorker worker; + + /** Total number of pages recycled from index trees. */ + private final AtomicLong pageCnt = new AtomicLong(); + + /** + * Constructor. + * + * @param grpName Cache group name. + * @param cacheName Cache name. + * @param idxName Index name. + * @param oldTreeName Old name of underlying index tree name. + * @param newTreeName New name of underlying index tree name. + * @param segments Number of segments. + * @param trees Index trees. + */ + public DurableBackgroundCleanupIndexTreeTaskV2( + @Nullable String grpName, + String cacheName, + String idxName, + String oldTreeName, + String newTreeName, + int segments, + @Nullable InlineIndexTree[] trees + ) { + uid = UUID.randomUUID().toString(); + this.grpName = grpName; + this.cacheName = cacheName; + this.idxName = idxName; + this.oldTreeName = oldTreeName; + this.newTreeName = newTreeName; + this.segments = segments; + + if (trees != null) { + assert trees.length == segments : + "Invalid number of index trees [trees=" + trees.length + ", segments=" + segments + ']'; + + Map<Integer, RootPage> rootPages0 = new ConcurrentHashMap<>(); + + for (int i = 0; i < trees.length; i++) { + InlineIndexTree tree = trees[i]; + + assert tree != null; + + tree.close(); + + rootPages0.put(i, rootPage(tree)); + } + + this.rootPages = rootPages0; + } + + needToRen = true; + } + + /** + * Default constructor for {@link Externalizable}. + */ + public DurableBackgroundCleanupIndexTreeTaskV2() { + // No-op. + } + + /** {@inheritDoc} */ + @Override protected void writeExternalData(ObjectOutput out) throws IOException { + U.writeLongString(out, uid); + U.writeLongString(out, grpName); + U.writeLongString(out, cacheName); + U.writeLongString(out, idxName); + U.writeLongString(out, oldTreeName); + U.writeLongString(out, newTreeName); + out.writeInt(segments); + } + + /** {@inheritDoc} */ + @Override protected void readExternalData( + byte protoVer, + ObjectInput in + ) throws IOException, ClassNotFoundException { + uid = U.readLongString(in); + grpName = U.readLongString(in); + cacheName = U.readLongString(in); + idxName = U.readLongString(in); + oldTreeName = U.readLongString(in); + newTreeName = U.readLongString(in); + segments = in.readInt(); + } + + /** {@inheritDoc} */ + @Override public String name() { + return "drop-sql-index-" + cacheName + "-" + idxName + "-" + uid; + } + + /** {@inheritDoc} */ + @Override public void cancel() { + rootPages = null; + + GridWorker w = worker; + + if (w != null) { + worker = null; + + U.awaitForWorkersStop(singleton(w), true, log); + } + } + + /** {@inheritDoc} */ + @Override public void onDeactivationCluster() { + rootPages = null; + } + + /** {@inheritDoc} */ + @Override public IgniteInternalFuture<DurableBackgroundTaskResult<Long>> executeAsync(GridKernalContext ctx) { + assert worker == null; + + log = ctx.log(DurableBackgroundCleanupIndexTreeTaskV2.class); + + IgniteInternalFuture<DurableBackgroundTaskResult<Long>> outFut; + + CacheGroupContext grpCtx = ctx.cache().cacheGroup(CU.cacheGroupId(cacheName, grpName)); + + if (grpCtx != null) { + try { + // Renaming should be done once when adding (and immediately launched) a task at the time of drop the index. + // To avoid problems due to node crash between renaming and adding a task. + if (needToRen) { + // If the node falls before renaming, then the index was definitely not dropped. + // If the node crashes after renaming, the task will delete the old index trees, + // and the node will rebuild this index when the node starts. + renameIndexRootPages(grpCtx, cacheName, oldTreeName, newTreeName, segments); + + // After restoring from MetaStorage, it will also be {@code false}. + needToRen = false; + } + + @Nullable Map<Integer, RootPage> rootPages0 = this.rootPages; + + if (rootPages0 == null) { + Map<Integer, RootPage> rootPages1 = findIndexRootPages(grpCtx, cacheName, newTreeName, segments); + + if (!rootPages1.isEmpty()) + this.rootPages = rootPages0 = new ConcurrentHashMap<>(rootPages1); + } + + if (!F.isEmpty(rootPages0)) { + GridFutureAdapter<DurableBackgroundTaskResult<Long>> fut = new GridFutureAdapter<>(); + + Map<Integer, RootPage> finalRootPages = rootPages0; + + GridWorker w = new GridWorker( + ctx.igniteInstanceName(), + "async-worker-" + name(), + log + ) { + /** {@inheritDoc} */ + @Override protected void body() { + try { + Iterator<Map.Entry<Integer, RootPage>> it = finalRootPages.entrySet().iterator(); + + while (it.hasNext()) { + Map.Entry<Integer, RootPage> e = it.next(); + + RootPage rootPage = e.getValue(); + int segment = e.getKey(); + + long pages = destroyIndexTrees(grpCtx, rootPage, cacheName, newTreeName, segment); + + if (pages > 0) + pageCnt.addAndGet(pages); + + it.remove(); + } + + fut.onDone(DurableBackgroundTaskResult.complete(pageCnt.get())); + } + catch (Throwable t) { + fut.onDone(DurableBackgroundTaskResult.restart(t)); + } + finally { + worker = null; + } + } + }; + + new IgniteThread(w).start(); + + this.worker = w; + + outFut = fut; + } + else + outFut = new GridFinishedFuture<>(DurableBackgroundTaskResult.complete()); + } + catch (Throwable t) { + outFut = new GridFinishedFuture<>(DurableBackgroundTaskResult.restart(t)); + } + } + else + outFut = new GridFinishedFuture<>(DurableBackgroundTaskResult.complete()); + + return outFut; + } + + /** + * Destroying index trees. + * + * @param grpCtx Cache group context. + * @param rootPage Index root page. + * @param cacheName Cache name. + * @param treeName Name of underlying index tree name. + * @param segment Segment number. + * @return Total number of pages recycled from this tree. + * @throws IgniteCheckedException If failed. + */ + public static long destroyIndexTrees( + CacheGroupContext grpCtx, + RootPage rootPage, + String cacheName, + String treeName, + int segment + ) throws IgniteCheckedException { + long pageCnt = 0; + + grpCtx.shared().database().checkpointReadLock(); + + try { + if (existMetaPage(grpCtx, rootPage.pageId().pageId())) { Review comment: What's the purpose of this check? Can it fail in the new implementation? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
