[ignite-3] branch main updated: IGNITE-19362 Fix deadlock while restarting with existing raft snapshot and scheduled index rebuild (#1983)

[sdanilov]

This is an automated email from the ASF dual-hosted git repository.

sdanilov pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/ignite-3.git


The following commit(s) were added to refs/heads/main by this push:
     new dcaa4da3ae IGNITE-19362 Fix deadlock while restarting with existing 
raft snapshot and scheduled index rebuild (#1983)
dcaa4da3ae is described below

commit dcaa4da3ae31578fe49efcab3a0bbde2142ba0d7
Author: Semyon Danilov <samvi...@yandex.ru>
AuthorDate: Wed Apr 26 12:02:08 2023 +0400

    IGNITE-19362 Fix deadlock while restarting with existing raft snapshot and 
scheduled index rebuild (#1983)
---
 .../storage/snapshot/SnapshotExecutorImpl.java     |  8 +++-
 .../runner/app/ItIgniteNodeRestartTest.java        | 49 ++++++++++++++++++++++
 2 files changed, 56 insertions(+), 1 deletion(-)

diff --git 
a/modules/raft/src/main/java/org/apache/ignite/raft/jraft/storage/snapshot/SnapshotExecutorImpl.java
 
b/modules/raft/src/main/java/org/apache/ignite/raft/jraft/storage/snapshot/SnapshotExecutorImpl.java
index 30d966add3..f005b3199a 100644
--- 
a/modules/raft/src/main/java/org/apache/ignite/raft/jraft/storage/snapshot/SnapshotExecutorImpl.java
+++ 
b/modules/raft/src/main/java/org/apache/ignite/raft/jraft/storage/snapshot/SnapshotExecutorImpl.java
@@ -250,7 +250,13 @@ public class SnapshotExecutorImpl implements 
SnapshotExecutor {
         final FirstSnapshotLoadDone done = new FirstSnapshotLoadDone(reader);
         Requires.requireTrue(this.fsmCaller.onSnapshotLoad(done));
         try {
-            done.waitForRun();
+            // TODO: IGNITE-19363 We want to avoid deadlock for now, but this 
is an ad-hoc decision.
+            // We don't wait for the partition's snapshot load closure to 
finish here.
+            if (!node.getNodeId().getGroupId().contains("part")) {
+                done.waitForRun();
+            } else {
+                done.status = Status.OK();
+            }
         }
         catch (final InterruptedException e) {
             LOG.warn("Wait for FirstSnapshotLoadDone run is interrupted.");
diff --git 
a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/ItIgniteNodeRestartTest.java
 
b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/ItIgniteNodeRestartTest.java
index 1fcc58cb61..c4eef0882a 100644
--- 
a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/ItIgniteNodeRestartTest.java
+++ 
b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/ItIgniteNodeRestartTest.java
@@ -107,6 +107,7 @@ import org.apache.ignite.internal.table.TableImpl;
 import org.apache.ignite.internal.table.distributed.TableManager;
 import org.apache.ignite.internal.table.distributed.TableMessageGroup;
 import 
org.apache.ignite.internal.table.distributed.raft.snapshot.outgoing.OutgoingSnapshotsManager;
+import org.apache.ignite.internal.table.distributed.storage.InternalTableImpl;
 import org.apache.ignite.internal.testframework.IgniteAbstractTest;
 import org.apache.ignite.internal.testframework.TestIgnitionManager;
 import org.apache.ignite.internal.testframework.WithSystemProperty;
@@ -878,6 +879,54 @@ public class ItIgniteNodeRestartTest extends 
IgniteAbstractTest {
         checkTableWithData(ignite, TABLE_NAME);
     }
 
+    /**
+     * Restarts the node which stores some data.
+     */
+    @Test
+    public void nodeWithDataAndIndexRebuildTest() throws InterruptedException {
+        IgniteImpl ignite = startNode(0);
+
+        int partitions = 20;
+
+        createTableWithData(List.of(ignite), TABLE_NAME, 1, partitions);
+
+        TableImpl table = (TableImpl) ignite.tables().table(TABLE_NAME);
+
+        InternalTableImpl internalTable = (InternalTableImpl) 
table.internalTable();
+
+        CompletableFuture[] flushFuts = new CompletableFuture[partitions];
+
+        for (int i = 0; i < partitions; i++) {
+            // Flush data on disk, so that we will have a snapshot to read on 
restart.
+            flushFuts[i] = internalTable.storage().getMvPartition(i).flush();
+        }
+
+        assertThat(CompletableFuture.allOf(flushFuts), 
willCompleteSuccessfully());
+
+        // Add more data, so that on restart there will be a index rebuilding 
operation.
+        try (Session session = ignite.sql().createSession()) {
+            for (int i = 0; i < 100; i++) {
+                session.execute(null, "INSERT INTO " + TABLE_NAME + "(id, 
name) VALUES (?, ?)",
+                        i + 500, VALUE_PRODUCER.apply(i + 500));
+            }
+        }
+
+        stopNode(0);
+
+        ignite = startNode(0);
+
+        checkTableWithData(ignite, TABLE_NAME);
+
+        table = (TableImpl) ignite.tables().table(TABLE_NAME);
+
+        // Check data that was added after flush.
+        for (int i = 0; i < 100; i++) {
+            Tuple row = table.keyValueView().get(null, 
Tuple.create().set("id", i + 500));
+
+            assertEquals(VALUE_PRODUCER.apply(i + 500), 
row.stringValue("name"));
+        }
+    }
+
     /**
      * Starts two nodes and checks that the data are storing through restarts. 
Nodes restart in the same order when they started at first.
      */