[
https://issues.apache.org/jira/browse/IGNITE-28874?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=18095008#comment-18095008
]
Mikhail Petrov commented on IGNITE-28874:
-----------------------------------------
[~kukushal] Could you please confirm that the issue you described is not
reproducible on the master branch?
I ran the attached example locally on the master branch and it completed
successfully.
The goal of the resolved IGNITE-28369 ticket was to fix the very similar (or
exactly the same) problem.
> Ignite service is never redeployed when its node fails
> ------------------------------------------------------
>
> Key: IGNITE-28874
> URL: https://issues.apache.org/jira/browse/IGNITE-28874
> Project: Ignite
> Issue Type: Bug
> Affects Versions: 2.18
> Reporter: Alexey Kukushkin
> Priority: Major
>
> It is expected that an Ignite service is always redeployed on another
> available node after its node fails. However, it does not happen if the next
> node that Ignite chooses for the service also fails during the service
> redeployment. See the reproducer below.
> {code:java}
> package sandbox.ignite;
> import org.apache.ignite.Ignite;
> import org.apache.ignite.Ignition;
> import org.apache.ignite.cluster.ClusterNode;
> import org.apache.ignite.configuration.IgniteConfiguration;
> import org.apache.ignite.failure.NoOpFailureHandler;
> import org.apache.ignite.lang.IgnitePredicate;
> import org.apache.ignite.services.Service;
> import org.apache.ignite.services.ServiceConfiguration;
> import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
> import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
> import org.junit.jupiter.api.RepeatedTest;
> import java.time.Duration;
> import java.util.Collections;
> import java.util.HashSet;
> import java.util.Optional;
> import java.util.Set;
> import static org.junit.jupiter.api.Assertions.assertTrue;
> public class CfAll44439Test {
> private final static int FAILURE_DETECTION_TIMEOUT = 2_000;
> /**
> * EXPECTED: An Ignite service is always redeployed on the next available
> node after its node fails.
> * ACTUAL: The service is never redeployed if the next available node
> chosen for it fails during
> * the service redeployment.
> */
> @RepeatedTest(10)
> public void serviceIsRedeployedAfterItsNodeFails() throws
> InterruptedException {
> // GIVEN an Ignite cluster consisting of 1 server and 3 client nodes
> with a 2-second failure detection timeout
> var aliveClients = new HashSet<>(Set.of("client1", "client2",
> "client3"));
> try (final var server1 =
> Ignition.start(getIgniteConfiguration("server1", false))) {
> for (final var clientName : aliveClients) {
> Ignition.start(getIgniteConfiguration(clientName, true));
> }
> // AND an Ignite service named "SingletonService" is deployed on
> one of the client nodes as a Cluster
> // Singleton. The "SingletonService" is initially deployed
> without any delays. However, redeploying
> // the service takes 4 times the failure detection timeout (8
> seconds).
> final var service = new SingletonService(Duration.ofMillis(4 *
> FAILURE_DETECTION_TIMEOUT));
> final var serviceName = SingletonService.class.getName();
> final var serviceConfig = new ServiceConfiguration()
> .setService(service)
> .setName(serviceName)
> .setNodeFilter(new AllClientsPredicate())
> .setMaxPerNodeCount(0)
> .setTotalCount(1);
> server1.services().deploy(serviceConfig);
> final var initialServiceNode =
> getServiceNodeConsistentId(server1, serviceName);
> assertTrue(
> initialServiceNode.isPresent(),
> "Initial setup: all clients are alive, the service is on " +
> initialServiceNode);
> // AND the node hosting the "SingletonService" stops
> Ignition.stop(initialServiceNode.orElseThrow(), true);
> aliveClients.remove(initialServiceNode.orElseThrow());
> var nextServiceNode = getServiceNodeConsistentId(server1,
> serviceName);
> assertTrue(
> nextServiceNode.isEmpty(),
> "The service is not deployed immediately after its node is
> killed since redeployment takes " +
> "(4 x timeout)");
> // AND 4 seconds passes
> Thread.sleep(2 * FAILURE_DETECTION_TIMEOUT);
> nextServiceNode = getServiceNodeConsistentId(server1,
> serviceName);
> assertTrue(
> nextServiceNode.isEmpty(),
> "The service is still not deployed after (2 x timeout) since
> redeployment takes (4 x timeout)");
> // AND the next node Ignite scheduled to deploy the service on
> stops
> // NOTE: at this moment the service is still not redeployed, but
> Ignite has already chosen the next
> // node for deployment. There is no API to query that node, so we
> pick one of the two remaining
> // client nodes at random. This means there is a 50% chance of
> picking the wrong node, and thus
> // the test reproduces the problem with 50% probability. The test
> is repeated to increase that probability.
> var nextKilledClient = aliveClients.iterator().next();
> Ignition.stop(nextKilledClient, true);
> aliveClients.remove(nextKilledClient);
> nextServiceNode = getServiceNodeConsistentId(server1,
> serviceName);
> assertTrue(
> nextServiceNode.isEmpty(),
> "The service is still not deployed immediately after another
> node is killed");
> // WHEN another 12 seconds pass, making the total time 16
> seconds, which is twice the
> // service's 8-second redeployment time.
> Thread.sleep(6 * FAILURE_DETECTION_TIMEOUT);
> // THEN the service is redeployed on the remaining client node
> nextServiceNode = getServiceNodeConsistentId(server1,
> serviceName);
> final var remainingClient = aliveClients.iterator().next();
> assertTrue(
> nextServiceNode.isPresent() &&
> nextServiceNode.orElseThrow().equals(remainingClient),
> "The service must be deployed on the remaining " +
> remainingClient + " after (8 x timeout) since " +
> "the elapsed time significantly exceeds the service's (4
> x timeout) redeployment time");
> Ignition.stop(remainingClient, true);
> }
> }
> private static Optional<String> getServiceNodeConsistentId(Ignite ignite,
> String serviceName) {
> return ignite.services().serviceDescriptors().stream()
> .filter(s -> s.name().equals(serviceName))
> .findFirst()
> .flatMap(descriptor -> descriptor
> .topologySnapshot()
> .keySet()
> .stream()
> .findFirst()
> .flatMap(uuid ->
> Optional.ofNullable(ignite.cluster().node(uuid)))
> .map(node -> node.consistentId().toString()));
> }
> private static IgniteConfiguration getIgniteConfiguration(final String
> igniteName, final boolean isClient) {
> return new IgniteConfiguration()
> .setClientMode(isClient)
> .setIgniteInstanceName(igniteName)
> .setConsistentId(igniteName)
> .setFailureDetectionTimeout(FAILURE_DETECTION_TIMEOUT)
> .setClientFailureDetectionTimeout(FAILURE_DETECTION_TIMEOUT)
> .setFailureHandler(new NoOpFailureHandler())
> .setDiscoverySpi(
> new TcpDiscoverySpi()
> .setIpFinder(new
> TcpDiscoveryVmIpFinder().setAddresses(Collections.singleton("127.0.0.1:49500")))
> .setLocalPort(49500)
> );
> }
> private static class SingletonService implements Service {
> private static boolean initiallyDeployed = false;
> private final Duration redeployDuration;
> private SingletonService(final Duration redeployDuration) {
> this.redeployDuration = redeployDuration;
> }
> @Override
> public void init() {
> try {
> if (initiallyDeployed) {
> Thread.sleep(redeployDuration);
> } else {
> initiallyDeployed = true;
> }
> } catch (InterruptedException e) {
> throw new RuntimeException(e);
> }
> }
> }
> private static class AllClientsPredicate implements
> IgnitePredicate<ClusterNode> {
> @Override
> public boolean apply(final ClusterNode clusterNode) {
> return clusterNode.isClient();
> }
> }
> }
> {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
