This is an automated email from the ASF dual-hosted git repository. ilyak pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push: new d75bfd0 IGNITE-13655 Implement readiness probe REST endpoint - Fixes #8417. d75bfd0 is described below commit d75bfd0f35d43ab29f9b280d31e9a0c3132da59d Author: akorensh <akore...@gmail.com> AuthorDate: Fri Nov 13 13:51:28 2020 +0300 IGNITE-13655 Implement readiness probe REST endpoint - Fixes #8417. Signed-off-by: Ilya Kasnacheev <ilya.kasnach...@gmail.com> --- .../internal/client/rest/GridProbeCommandTest.java | 224 +++++++++++++++++++++ .../client/suite/IgniteClientTestSuite.java | 4 + .../org/apache/ignite/internal/IgnitionEx.java | 17 ++ .../internal/processors/rest/GridRestCommand.java | 5 +- .../processors/rest/GridRestProcessor.java | 2 + .../internal/processors/rest/GridRestResponse.java | 3 + .../handlers/probe/GridProbeCommandHandler.java | 70 +++++++ .../protocols/http/jetty/GridJettyRestHandler.java | 5 +- 8 files changed, 327 insertions(+), 3 deletions(-) diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/client/rest/GridProbeCommandTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/client/rest/GridProbeCommandTest.java new file mode 100644 index 0000000..af09544 --- /dev/null +++ b/modules/clients/src/test/java/org/apache/ignite/internal/client/rest/GridProbeCommandTest.java @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.client.rest; + +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.ignite.configuration.ConnectorConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.processors.rest.GridRestCommand; +import org.apache.ignite.internal.processors.rest.GridRestResponse; +import org.apache.ignite.internal.processors.rest.handlers.GridRestCommandHandler; +import org.apache.ignite.internal.processors.rest.handlers.probe.GridProbeCommandHandler; +import org.apache.ignite.internal.processors.rest.request.GridRestCacheRequest; +import org.apache.ignite.plugin.AbstractTestPluginProvider; +import org.apache.ignite.plugin.PluginProvider; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; + +/** + * Test whether REST probe command works correctly when kernal has started and vice versa. + */ +public class GridProbeCommandTest extends GridCommonAbstractTest { + /** */ + private static final int JETTY_PORT = 8080; + + /** */ + private CountDownLatch triggerRestCmdLatch = new CountDownLatch(1); + + /** */ + private CountDownLatch triggerPluginStartLatch = new CountDownLatch(1); + + /** */ + public static Map<String, Object> executeProbeRestRequest() throws IOException { + HttpURLConnection conn = (HttpURLConnection)(new URL("http://localhost:" + JETTY_PORT + "/ignite?cmd=probe").openConnection()); + conn.connect(); + + boolean isHTTP_OK = conn.getResponseCode() == HttpURLConnection.HTTP_OK; + + Map<String, Object> restResponse = null; + + try (InputStreamReader streamReader = new InputStreamReader(isHTTP_OK ? conn.getInputStream() : conn.getErrorStream())) { + + ObjectMapper objMapper = new ObjectMapper(); + restResponse = objMapper.readValue(streamReader, + new TypeReference<Map<String, Object>>() { + }); + + log.info("probe command response is: " + restResponse); + + } + catch (Exception e) { + log.error("error executing probe rest command", e); + } + return restResponse; + + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + cfg.setConnectorConfiguration(new ConnectorConfiguration()); + + if (igniteInstanceName.equals("regular")) + return cfg; + else if (igniteInstanceName.equals("delayedStart")) { + PluginProvider delayedStartPluginProvider = new DelayedStartPluginProvider(triggerPluginStartLatch, triggerRestCmdLatch); + + cfg.setPluginProviders(new PluginProvider[] {delayedStartPluginProvider}); + } + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + stopAllGrids(false); + } + + /** + * Test for the REST probe command + * + * @throws Exception If failed. + */ + @Test + public void testRestProbeCommand() throws Exception { + startGrid("regular"); + + GridRestCommandHandler hnd = new GridProbeCommandHandler((grid("regular")).context()); + + GridRestCacheRequest req = new GridRestCacheRequest(); + req.command(GridRestCommand.PROBE); + + IgniteInternalFuture<GridRestResponse> resp = hnd.handleAsync(req); + resp.get(); + + assertEquals(GridRestResponse.STATUS_SUCCESS, resp.result().getSuccessStatus()); + assertEquals("grid has started", resp.result().getResponse()); + + } + + /** + * <p>Test rest cmd=probe command given a non fully started kernal. </p> + * <p>1. start the grid on a seperate thread w/a plugin that will keep it waiting, at a point after rest http + * processor is ready, until signaled to proceed. </p> + * <p>2. when the grid.start() has reached the plugin init method(rest http processor has started now), issue a + * rest command against the non-fully started kernal. </p> + * <p>3. validate that the probe cmd has returned the appropriate erroneous code and message. </p> + * <p>4. stop the grid. </p> + * + * @throws Exception If failed. + */ + @Test + public void testRestProbeCommandGridNotStarted() throws Exception { + new Thread(new Runnable() { + @Override public void run() { + try { + startGrid("delayedStart"); + } + catch (Exception e) { + log.error("error when starting delatedStart grid", e); + } + } + }).start(); + + Map<String, Object> probeRestCommandResponse; + + log.info("awaiting plugin handler latch"); + triggerPluginStartLatch.await(); + log.info("starting rest command url call"); + try { + probeRestCommandResponse = executeProbeRestRequest(); + log.info("finished rest command url call"); + } + finally { + triggerRestCmdLatch.countDown(); //make sure the grid shuts down + } + + assertTrue(probeRestCommandResponse.get("error").equals("grid has not started")); + assertEquals(GridRestResponse.SERVICE_UNAVAILABLE, probeRestCommandResponse.get("successStatus")); + } + + /** + * <p>Start a regular grid, issue a cmd=probe rest command, and validate restponse + * + * @throws Exception If failed. + */ + @Test + public void testRestProbeCommandGridStarted() throws Exception { + startGrid("regular"); + + Map<String, Object> probeRestCommandResponse; + + probeRestCommandResponse = executeProbeRestRequest(); + + assertTrue(probeRestCommandResponse.get("response").equals("grid has started")); + assertEquals(0, probeRestCommandResponse.get("successStatus")); + } + + /** + * This plugin awaits until it is given the signal to process -- thereby allowing an http request against a non + * fully started kernal. + */ + public static class DelayedStartPluginProvider extends AbstractTestPluginProvider { + /** */ + private CountDownLatch triggerRestCmd; + + /** */ + private CountDownLatch triggerPluginStart; + + /** */ + public DelayedStartPluginProvider(CountDownLatch triggerPluginStartLatch, + CountDownLatch triggerRestCmdLatch) { + this.triggerPluginStart = triggerPluginStartLatch; + this.triggerRestCmd = triggerRestCmdLatch; + } + + /** {@inheritDoc} */ + @Override public String name() { + return "DelayedStartPlugin"; + } + + /** {@inheritDoc} */ + @Override public void onIgniteStart() { + super.onIgniteStart(); + + triggerPluginStart.countDown(); + + log.info("awaiting rest command latch ..."); + + try { + triggerRestCmd.await(); + } + catch (InterruptedException e) { + log.error("error in custom plugin", e); + } + + log.info("finished awaiting rest command latch."); + } + } +} diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/client/suite/IgniteClientTestSuite.java b/modules/clients/src/test/java/org/apache/ignite/internal/client/suite/IgniteClientTestSuite.java index 8a3936b..7408f4e 100644 --- a/modules/clients/src/test/java/org/apache/ignite/internal/client/suite/IgniteClientTestSuite.java +++ b/modules/clients/src/test/java/org/apache/ignite/internal/client/suite/IgniteClientTestSuite.java @@ -42,6 +42,7 @@ import org.apache.ignite.internal.client.integration.ClientTcpSslDirectSelfTest; import org.apache.ignite.internal.client.integration.ClientTcpSslMultiNodeSelfTest; import org.apache.ignite.internal.client.integration.ClientTcpSslSelfTest; import org.apache.ignite.internal.client.integration.ClientTcpUnreachableMultiNodeSelfTest; +import org.apache.ignite.internal.client.rest.GridProbeCommandTest; import org.apache.ignite.internal.client.router.ClientFailedInitSelfTest; import org.apache.ignite.internal.client.router.RouterFactorySelfTest; import org.apache.ignite.internal.client.router.TcpRouterMultiNodeSelfTest; @@ -134,6 +135,9 @@ import org.junit.runners.Suite; ClientTcpUnreachableMultiNodeSelfTest.class, ClientPreferDirectSelfTest.class, + //Test REST probe cmd + GridProbeCommandTest.class, + // Test client with many nodes and in multithreaded scenarios ClientTcpMultiThreadedSelfTest.class, ClientTcpSslMultiThreadedSelfTest.class, diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java index dfdae46..3104269 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java @@ -1477,6 +1477,16 @@ public class IgnitionEx { } /** + * @param name Grid name (possibly {@code null} for default grid). + * @return true when all managers, processors, and plugins have started and ignite kernal start method has fully + * completed. + */ + public static boolean hasKernalStarted(String name) { + IgniteNamedInstance grid = name != null ? grids.get(name) : dfltGrid; + return grid != null && grid.hasStartLatchCompleted(); + } + + /** * Start context encapsulates all starting parameters. */ private static final class GridStartContext { @@ -3215,6 +3225,13 @@ public class IgnitionEx { this.cnt = cnt; } } + + /** + * @return whether the startLatch has been counted down, thereby indicating that the kernal has full started. + */ + public boolean hasStartLatchCompleted() { + return startLatch.getCount() == 0; + } } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestCommand.java index c97c26a..16dc5f0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestCommand.java @@ -223,7 +223,10 @@ public enum GridRestCommand { NODE_STATE_BEFORE_START("nodestatebeforestart"), /** Warm-up. */ - WARM_UP("warmup"); + WARM_UP("warmup"), + + /** probe. */ + PROBE("probe"); /** Enum values. */ private static final GridRestCommand[] VALS = values(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java index 358f75b..21c5eb3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java @@ -56,6 +56,7 @@ import org.apache.ignite.internal.processors.rest.handlers.cluster.GridClusterNa import org.apache.ignite.internal.processors.rest.handlers.datastructures.DataStructuresCommandHandler; import org.apache.ignite.internal.processors.rest.handlers.log.GridLogCommandHandler; import org.apache.ignite.internal.processors.rest.handlers.memory.MemoryMetricsCommandHandler; +import org.apache.ignite.internal.processors.rest.handlers.probe.GridProbeCommandHandler; import org.apache.ignite.internal.processors.rest.handlers.query.QueryCommandHandler; import org.apache.ignite.internal.processors.rest.handlers.task.GridTaskCommandHandler; import org.apache.ignite.internal.processors.rest.handlers.top.GridTopologyCommandHandler; @@ -557,6 +558,7 @@ public class GridRestProcessor extends GridProcessorAdapter implements IgniteRes addHandler(new GridBaselineCommandHandler(ctx)); addHandler(new MemoryMetricsCommandHandler(ctx)); addHandler(new NodeStateBeforeStartCommandHandler(ctx)); + addHandler(new GridProbeCommandHandler(ctx)); // Start protocols. startTcpProtocol(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestResponse.java index 0c3ac04..adefd9e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestResponse.java @@ -51,6 +51,9 @@ public class GridRestResponse implements Externalizable { /** Success status. */ private int successStatus = STATUS_SUCCESS; + /** HTTP REQUEST not allowed */ + public static final int SERVICE_UNAVAILABLE = 503; + /** Session token. */ private byte[] sesTokBytes; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/probe/GridProbeCommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/probe/GridProbeCommandHandler.java new file mode 100644 index 0000000..844dd5b --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/probe/GridProbeCommandHandler.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.rest.handlers.probe; + +import java.util.Collection; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.IgnitionEx; +import org.apache.ignite.internal.processors.rest.GridRestCommand; +import org.apache.ignite.internal.processors.rest.GridRestResponse; +import org.apache.ignite.internal.processors.rest.handlers.GridRestCommandHandlerAdapter; +import org.apache.ignite.internal.processors.rest.request.GridRestRequest; +import org.apache.ignite.internal.util.future.GridFinishedFuture; +import org.apache.ignite.internal.util.typedef.internal.U; + +import static org.apache.ignite.internal.processors.rest.GridRestCommand.PROBE; + +/** + * Handler for {@link GridRestCommand#PROBE}. + */ +public class GridProbeCommandHandler extends GridRestCommandHandlerAdapter { + /** + * @param ctx Context. + */ + public GridProbeCommandHandler(GridKernalContext ctx) { + super(ctx); + } + + /** Supported commands. */ + private static final Collection<GridRestCommand> SUPPORTED_COMMANDS = U.sealList(PROBE); + + /** {@inheritDoc} */ + @Override public Collection<GridRestCommand> supportedCommands() { + return SUPPORTED_COMMANDS; + } + + /** {@inheritDoc} */ + @Override public IgniteInternalFuture<GridRestResponse> handleAsync(GridRestRequest req) { + assert req != null; + + assert SUPPORTED_COMMANDS.contains(req.command()); + + switch (req.command()) { + case PROBE: { + if (log.isDebugEnabled()) + log.debug("probe command handler invoked."); + + return new GridFinishedFuture<>(IgnitionEx.hasKernalStarted(ctx.igniteInstanceName()) ? new GridRestResponse("grid has started") : new GridRestResponse(GridRestResponse.SERVICE_UNAVAILABLE, "grid has not started")); + + } + } + + return new GridFinishedFuture<>(); + } +} diff --git a/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java b/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java index 42b8a30..421bf78 100644 --- a/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java +++ b/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java @@ -464,7 +464,7 @@ public class GridJettyRestHandler extends AbstractHandler { if (sesTok != null) cmdRes.setSessionToken(U.byteArray2HexString(sesTok)); - res.setStatus(HttpServletResponse.SC_OK); + res.setStatus(cmdRes.getSuccessStatus() == GridRestResponse.SERVICE_UNAVAILABLE ? HttpServletResponse.SC_SERVICE_UNAVAILABLE : HttpServletResponse.SC_OK); } catch (Throwable e) { res.setStatus(HttpServletResponse.SC_OK); @@ -720,7 +720,8 @@ public class GridJettyRestHandler extends AbstractHandler { case DATA_REGION_METRICS: case DATA_STORAGE_METRICS: case NAME: - case VERSION: { + case VERSION: + case PROBE: { restReq = new GridRestRequest(); break;