This is an automated email from the ASF dual-hosted git repository.
akuznetsov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 0971768 IGNITE-13081 Fix rest processor hanging on node shutdown if
start failed #7859 - Fixes #7859.
0971768 is described below
commit 09717684fb20e53afd984182da0acbf62eaf7dc3
Author: Semyon Danilov <[email protected]>
AuthorDate: Mon Jun 1 13:35:31 2020 +0700
IGNITE-13081 Fix rest processor hanging on node shutdown if start failed
#7859 - Fixes #7859.
Signed-off-by: Alexey Kuznetsov <[email protected]>
---
.../processors/rest/GridRestProcessor.java | 15 ++-
.../processors/rest/RestProcessorHangTest.java | 148 +++++++++++++++++++++
.../testsuites/IgniteRestHandlerTestSuite.java | 4 +-
3 files changed, 159 insertions(+), 8 deletions(-)
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java
index 0c4797e..02c8658 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java
@@ -607,14 +607,15 @@ public class GridRestProcessor extends
GridProcessorAdapter implements IgniteRes
boolean interrupted = Thread.interrupted();
- while (workersCnt.sum() != 0) {
- try {
- Thread.sleep(200);
- }
- catch (InterruptedException ignored) {
- interrupted = true;
+ if (!cancel)
+ while (workersCnt.sum() != 0) {
+ try {
+ Thread.sleep(200);
+ }
+ catch (InterruptedException ignored) {
+ interrupted = true;
+ }
}
- }
U.interrupt(sesTimeoutCheckerThread);
diff --git
a/modules/core/src/test/java/org/apache/ignite/internal/processors/rest/RestProcessorHangTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/rest/RestProcessorHangTest.java
new file mode 100644
index 0000000..124934a
--- /dev/null
+++
b/modules/core/src/test/java/org/apache/ignite/internal/processors/rest/RestProcessorHangTest.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.rest;
+
+import java.io.IOException;
+import java.net.Socket;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.CountDownLatch;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.ConnectorConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.IgnitionEx;
+import org.apache.ignite.internal.processors.rest.request.GridRestCacheRequest;
+import org.apache.ignite.spi.discovery.tcp.TestTcpDiscoverySpi;
+import
org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Test;
+
+/**
+ * Test for rest processor hanging on stop.
+ */
+public class RestProcessorHangTest extends GridCommonAbstractTest {
+ /**
+ * {@inheritDoc}
+ */
+ @Override protected IgniteConfiguration getConfiguration(String gridName)
throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ ConnectorConfiguration connectorConfiguration = new
ConnectorConfiguration();
+
+ cfg.setConnectorConfiguration(connectorConfiguration);
+
+ return cfg;
+ }
+
+ /**
+ * Test that node doesn't hang if there are rest requests and discovery
SPI failed.
+ *
+ * Description: Fire up one node that always rejects connections.
+ * Fire up another node and without waiting for it to start up publish
CACHE_GET request to the rest processor.
+ * This request will hang until the node start. As soon as this node fails
to connect to the first one it should
+ * stop itself causing RestProcessor to wait indefinitely for all
rest-workers to finish which leads to node
+ * hang on stop process.
+ */
+ @Test
+ public void testNodeStopOnDiscoverySpiFailTest() throws Exception {
+ final CountDownLatch latch = new CountDownLatch(1);
+
+ String rejectorGridName = "rejector";
+
+ IgniteConfiguration regjectorGridCfg =
getConfiguration(rejectorGridName);
+
+ // Discovery spi that never allows connecting.
+ TestTcpDiscoverySpi discoSpi = new TestTcpDiscoverySpi() {
+ @Override protected void writeToSocket(TcpDiscoveryAbstractMessage
msg, Socket sock, int res, long timeout) throws IOException {
+ try {
+ // Wait until request is added to rest processor.
+ latch.await();
+ }
+ catch (InterruptedException ignored) {
+ // No-op.
+ }
+
+ super.writeToSocket(msg, sock, 255, timeout);
+ }
+ };
+
+ discoSpi.setIpFinder(sharedStaticIpFinder);
+
+ regjectorGridCfg.setDiscoverySpi(discoSpi);
+
+ startGrid(regjectorGridCfg);
+
+ String hangGridName = "impossibleToJoin";
+
+ IgniteConfiguration hangNodeCfg = getConfiguration(hangGridName);
+
+ GridTestUtils.runAsync(() -> startGrid(hangNodeCfg));
+
+ GridTestUtils.waitForCondition(() -> {
+ try {
+ IgniteKernal failingGrid = IgnitionEx.gridx(hangGridName);
+
+ return failingGrid != null && failingGrid.context().rest() !=
null;
+ }
+ catch (Exception ignored) {
+ return false;
+ }
+ }, 20_000);
+
+ IgniteEx hangGrid = IgnitionEx.gridx(hangGridName);
+
+ IgniteRestProcessor rest = hangGrid.context().rest();
+
+ new Thread(() -> {
+ GridRestProtocolHandler hnd = GridTestUtils.getFieldValue(rest,
"protoHnd");
+
+ GridRestCacheRequest req = new GridRestCacheRequest();
+
+ req.cacheName(DEFAULT_CACHE_NAME);
+
+ req.command(GridRestCommand.CACHE_GET);
+
+ req.key("k1");
+
+ latch.countDown();
+
+ try {
+ // Submitting cache get request to node that didn't fully
start must hang.
+ hnd.handle(req);
+ }
+ catch (IgniteCheckedException ignored) {
+ // No-op.
+ }
+ }).start();
+
+ latch.await();
+
+ // Node should stop correctly.
+ assertTrue(GridTestUtils.waitForCondition(() -> {
+ List<Ignite> ignites = IgnitionEx.allGrids();
+ return ignites.stream().noneMatch(ignite ->
Objects.equals(ignite.name(), hangGridName));
+ }, 20_000));
+
+ stopGrid(rejectorGridName);
+ }
+
+}
diff --git
a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteRestHandlerTestSuite.java
b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteRestHandlerTestSuite.java
index b2369a8..bfcb33c 100644
---
a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteRestHandlerTestSuite.java
+++
b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteRestHandlerTestSuite.java
@@ -18,6 +18,7 @@
package org.apache.ignite.testsuites;
import
org.apache.ignite.internal.processors.rest.RestProcessorInitializationTest;
+import org.apache.ignite.internal.processors.rest.RestProcessorHangTest;
import org.apache.ignite.internal.processors.rest.RestProtocolStartTest;
import
org.apache.ignite.internal.processors.rest.handlers.cache.GridCacheAtomicCommandHandlerSelfTest;
import
org.apache.ignite.internal.processors.rest.handlers.cache.GridCacheCommandHandlerSelfTest;
@@ -38,7 +39,8 @@ import org.junit.runners.Suite;
GridQueryCommandHandlerTest.class,
CacheTopologyCommandHandlerTest.class,
RestProtocolStartTest.class,
- RestProcessorInitializationTest.class
+ RestProcessorInitializationTest.class,
+ RestProcessorHangTest.class
})
public class IgniteRestHandlerTestSuite {
}
