This is an automated email from the ASF dual-hosted git repository.
mpetrov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new afff12420dd IGNITE-27216 Added capturing of cluster node certificates
during join process (#12546)
afff12420dd is described below
commit afff12420dd819b66c162bfd0e4b4d74e778414a
Author: Mikhail Petrov <[email protected]>
AuthorDate: Wed Dec 10 13:54:24 2025 +0300
IGNITE-27216 Added capturing of cluster node certificates during join
process (#12546)
---
.../ignite/internal/IgniteNodeAttributes.java | 3 +
.../ignite/spi/discovery/tcp/ClientImpl.java | 5 +
.../ignite/spi/discovery/tcp/ServerImpl.java | 22 ++-
.../ignite/spi/discovery/tcp/TcpDiscoveryImpl.java | 13 ++
.../spi/discovery/tcp/TcpDiscoveryIoSession.java | 19 +++
.../NodeConnectionCertificateCapturingTest.java | 190 +++++++++++++++++++++
.../ignite/testsuites/SecurityTestSuite.java | 2 +
7 files changed, 253 insertions(+), 1 deletion(-)
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java
b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java
index 7e79a695cae..a5fc9270d40 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java
@@ -150,6 +150,9 @@ public final class IgniteNodeAttributes {
/** V2 security subject for authenticated node. */
public static final String ATTR_SECURITY_SUBJECT_V2 = ATTR_PREFIX +
".security.subject.v2";
+ /** Node certificates the connection was established with. */
+ public static final String ATTR_NODE_CERTIFICATES = ATTR_PREFIX +
".security.certificates";
+
/** Client mode flag. */
public static final String ATTR_CLIENT_MODE = ATTR_PREFIX +
".cache.client";
diff --git
a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 094d44ac1ca..2fbb44bad84 100644
---
a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++
b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -2301,6 +2301,9 @@ class ClientImpl extends TcpDiscoveryImpl {
}
locNode.setAttributes(msg.clientNodeAttributes());
+
+ clearNodeSensitiveData(locNode);
+
locNode.visible(true);
long topVer = msg.topologyVersion();
@@ -2356,6 +2359,8 @@ class ClientImpl extends TcpDiscoveryImpl {
assert topVer > 0 : msg;
if (!node.visible()) {
+ clearNodeSensitiveData(node);
+
node.order(topVer);
node.visible(true);
diff --git
a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 9c767732cc5..da2c1e32b2a 100644
---
a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++
b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -178,6 +178,7 @@ import static
org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER;
import static
org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_COMPACT_FOOTER;
import static
org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2;
import static
org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_DFLT_SUID;
+import static
org.apache.ignite.internal.IgniteNodeAttributes.ATTR_NODE_CERTIFICATES;
import static
org.apache.ignite.internal.cluster.DistributedConfigurationUtils.CONN_DISABLED_BY_ADMIN_ERR_MSG;
import static
org.apache.ignite.internal.cluster.DistributedConfigurationUtils.newConnectionEnabledProperty;
import static
org.apache.ignite.internal.processors.security.SecurityUtils.authenticateLocalNode;
@@ -1134,7 +1135,7 @@ class ServerImpl extends TcpDiscoveryImpl {
leavingNodes.clear();
failedNodesMsgSent.clear();
-
locNode.attributes().remove(IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS);
+ clearNodeSensitiveData(locNode);
locNode.order(1);
locNode.internalOrder(1);
@@ -2443,6 +2444,18 @@ class ServerImpl extends TcpDiscoveryImpl {
}
}
+ /** */
+ private static void enrichNodeWithAttribute(TcpDiscoveryNode node, String
attrName, @Nullable Object attrVal) {
+ if (attrVal == null)
+ return;
+
+ Map<String, Object> attrs = new HashMap<>(node.getAttributes());
+
+ attrs.put(attrName, attrVal);
+
+ node.setAttributes(attrs);
+ }
+
/** */
private static WorkersRegistry getWorkerRegistry(TcpDiscoverySpi spi) {
return spi.ignite() instanceof IgniteEx ?
((IgniteEx)spi.ignite()).context().workersRegistry() : null;
@@ -5298,6 +5311,8 @@ class ServerImpl extends TcpDiscoveryImpl {
if (msg.verified()) {
assert topVer > 0 : "Invalid topology version: " + msg;
+ clearNodeSensitiveData(node);
+
if (node.order() == 0)
node.order(topVer);
@@ -7072,6 +7087,11 @@ class ServerImpl extends TcpDiscoveryImpl {
else if (msg instanceof
TcpDiscoveryJoinRequestMessage) {
TcpDiscoveryJoinRequestMessage req =
(TcpDiscoveryJoinRequestMessage)msg;
+ // Current node holds connection with the node
that is joining the cluster. Therefore, it can
+ // save certificates with which the connection was
established to joining node attributes.
+ if (spi.nodeAuth != null &&
nodeId.equals(req.node().id()))
+ enrichNodeWithAttribute(req.node(),
ATTR_NODE_CERTIFICATES, ses.extractCertificates());
+
if (!req.responded()) {
boolean ok = processJoinRequestMessage(req,
clientMsgWrk);
diff --git
a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
index 6ff2faa1974..f35f4114795 100644
---
a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
+++
b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
@@ -25,6 +25,7 @@ import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
@@ -53,6 +54,8 @@ import org.jetbrains.annotations.Nullable;
import static
org.apache.ignite.IgniteSystemProperties.IGNITE_DISCOVERY_METRICS_QNT_WARN;
import static org.apache.ignite.IgniteSystemProperties.getInteger;
+import static
org.apache.ignite.internal.IgniteNodeAttributes.ATTR_NODE_CERTIFICATES;
+import static
org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS;
import static
org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi.DFLT_DISCOVERY_METRICS_QNT_WARN;
/**
@@ -401,6 +404,16 @@ abstract class TcpDiscoveryImpl {
return true;
}
+ /** */
+ protected void clearNodeSensitiveData(TcpDiscoveryNode node) {
+ Map<String, Object> attrs = new HashMap<>(node.attributes());
+
+ attrs.remove(ATTR_NODE_CERTIFICATES);
+ attrs.remove(ATTR_SECURITY_CREDENTIALS);
+
+ node.setAttributes(attrs);
+ }
+
/** */
public void processMsgCacheMetrics(TcpDiscoveryMetricsUpdateMessage msg,
long tsNanos) {
for (Map.Entry<UUID, TcpDiscoveryMetricsUpdateMessage.MetricsSet> e :
msg.metrics().entrySet()) {
diff --git
a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryIoSession.java
b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryIoSession.java
index 88052bae9ee..5c1946f0eac 100644
---
a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryIoSession.java
+++
b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryIoSession.java
@@ -27,6 +27,9 @@ import java.io.OutputStream;
import java.io.StreamCorruptedException;
import java.net.Socket;
import java.nio.ByteBuffer;
+import java.security.cert.Certificate;
+import javax.net.ssl.SSLPeerUnverifiedException;
+import javax.net.ssl.SSLSocket;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.internal.direct.DirectMessageReader;
@@ -36,6 +39,7 @@ import org.apache.ignite.marshaller.jdk.JdkMarshaller;
import org.apache.ignite.plugin.extensions.communication.Message;
import org.apache.ignite.plugin.extensions.communication.MessageSerializer;
import
org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
+import org.jetbrains.annotations.Nullable;
import static
org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.makeMessageType;
@@ -206,6 +210,21 @@ public class TcpDiscoveryIoSession {
}
}
+ /** @return SSL certificate this session is established with. {@code null}
if SSL is disabled or certificate validation failed. */
+ @Nullable Certificate[] extractCertificates() {
+ if (!spi.isSslEnabled())
+ return null;
+
+ try {
+ return ((SSLSocket)sock).getSession().getPeerCertificates();
+ }
+ catch (SSLPeerUnverifiedException e) {
+ U.error(spi.log, "Failed to extract discovery IO session
certificates", e);
+
+ return null;
+ }
+ }
+
/**
* Serializes a discovery message into a byte array.
*
diff --git
a/modules/core/src/test/java/org/apache/ignite/internal/processors/security/NodeConnectionCertificateCapturingTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/security/NodeConnectionCertificateCapturingTest.java
new file mode 100644
index 00000000000..3a83133055d
--- /dev/null
+++
b/modules/core/src/test/java/org/apache/ignite/internal/processors/security/NodeConnectionCertificateCapturingTest.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.security;
+
+import java.security.Permissions;
+import java.security.cert.Certificate;
+import java.security.cert.X509Certificate;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.CountDownLatch;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.security.impl.TestSecurityData;
+import
org.apache.ignite.internal.processors.security.impl.TestSecurityPluginProvider;
+import
org.apache.ignite.internal.processors.security.impl.TestSecurityProcessor;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.plugin.security.SecurityCredentials;
+import org.apache.ignite.plugin.security.SecurityPermissionSet;
+import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.junit.Test;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_RECONNECTED;
+import static
org.apache.ignite.internal.IgniteNodeAttributes.ATTR_NODE_CERTIFICATES;
+import static
org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS;
+import static
org.apache.ignite.plugin.security.SecurityPermission.JOIN_AS_SERVER;
+import static
org.apache.ignite.plugin.security.SecurityPermissionSetBuilder.NO_PERMISSIONS;
+import static
org.apache.ignite.plugin.security.SecurityPermissionSetBuilder.systemPermissions;
+
+/** */
+public class NodeConnectionCertificateCapturingTest extends
AbstractSecurityTest {
+ /** */
+ private static final Collection<AuthenticationEvent>
NODE_AUTHENTICATION_EVENTS = new ConcurrentLinkedQueue<>();
+
+ /** */
+ @Test
+ public void testNodeConnectionCertificateCapturing() throws Exception {
+ checkNewNodeAuthenticationByClusterNodes(0, false, 1);
+ checkNewNodeAuthenticationByClusterNodes(1, false, 2);
+ checkNewNodeAuthenticationByClusterNodes(2, false, 3);
+ checkNewNodeAuthenticationByClusterNodes(3, true, 3);
+
+ // Checks nodes restart.
+ stopGrid(2);
+ stopGrid(3);
+
+ checkNewNodeAuthenticationByClusterNodes(3, true, 2);
+ checkNewNodeAuthenticationByClusterNodes(2, false, 3);
+
+ // Checks client node reconnect.
+ NODE_AUTHENTICATION_EVENTS.clear();
+
+ CountDownLatch cliNodeReconnectedLatch = new CountDownLatch(1);
+
+ grid(3).events().localListen(evt -> {
+ cliNodeReconnectedLatch.countDown();
+
+ return true;
+ }, EVT_CLIENT_NODE_RECONNECTED);
+
+ grid(0).context().discovery().failNode(grid(3).localNode().id(),
"test");
+
+ assertTrue(cliNodeReconnectedLatch.await(getTestTimeout(),
MILLISECONDS));
+
+ checkNodeAuthenticationByClusterNodes(3, grid(3).localNode().id(),
true, 3);
+ }
+
+ /** */
+ private void checkNewNodeAuthenticationByClusterNodes(int authNodeIdx,
boolean isClient, int expAuthCnt) throws Exception {
+ NODE_AUTHENTICATION_EVENTS.clear();
+
+ UUID authNodeId = startGrid(authNodeIdx,
isClient).cluster().localNode().id();
+
+ checkNodeAuthenticationByClusterNodes(authNodeIdx, authNodeId,
isClient, expAuthCnt);
+ }
+
+ /** */
+ private void checkNodeAuthenticationByClusterNodes(int authNodeIdx, UUID
authNodeId, boolean isClient, int expAuthCnt) {
+ assertEquals(expAuthCnt, NODE_AUTHENTICATION_EVENTS.size());
+
+ for (AuthenticationEvent auth : NODE_AUTHENTICATION_EVENTS) {
+ if (auth.clusterNodeId.equals(authNodeId))
+ assertNull(auth.certs);
+ else {
+ assertEquals(2, auth.certs.length);
+
+ X509Certificate cert = (X509Certificate)auth.certs[0];
+
+ assertEquals(isClient ? "CN=client" : "CN=node0" +
(authNodeIdx + 1), cert.getSubjectDN().getName());
+ }
+ }
+
+ for (Ignite ignite : G.allGrids()) {
+ for (ClusterNode node : ignite.cluster().nodes()) {
+
assertNull(((TcpDiscoveryNode)node).getAttributes().get(ATTR_SECURITY_CREDENTIALS));
+
assertNull(((TcpDiscoveryNode)node).getAttributes().get(ATTR_NODE_CERTIFICATES));
+ }
+ }
+ }
+
+ /** */
+ private IgniteEx startGrid(int idx, boolean isClient) throws Exception {
+ String login = getTestIgniteInstanceName(idx);
+
+ IgniteConfiguration cfg = getConfiguration(
+ login,
+ new SecurityPluginProvider(
+ login,
+ "",
+ isClient ? NO_PERMISSIONS : systemPermissions(JOIN_AS_SERVER),
+ null,
+ true
+ )).setClientMode(isClient);
+
+ cfg.setSslContextFactory(GridTestUtils.sslTrustedFactory(isClient ?
"client" : "node0" + (idx + 1), "trustboth"));
+
+ return startGrid(cfg);
+ }
+
+ /** */
+ private static class SecurityPluginProvider extends
TestSecurityPluginProvider {
+ /** */
+ SecurityPluginProvider(
+ String login,
+ String pwd,
+ SecurityPermissionSet perms,
+ Permissions sandboxPerms,
+ boolean globalAuth,
+ TestSecurityData... clientData
+ ) {
+ super(login, pwd, perms, sandboxPerms, globalAuth, clientData);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected GridSecurityProcessor
securityProcessor(GridKernalContext ctx) {
+ return new TestSecurityProcessor(
+ ctx,
+ new TestSecurityData(login, pwd, perms, sandboxPerms),
+ Arrays.asList(clientData),
+ globalAuth
+ ) {
+ @Override public SecurityContext authenticateNode(
+ ClusterNode node,
+ SecurityCredentials cred
+ ) throws IgniteCheckedException {
+ NODE_AUTHENTICATION_EVENTS.add(new
AuthenticationEvent(ctx.localNodeId(), node.attribute(ATTR_NODE_CERTIFICATES)));
+
+ return super.authenticateNode(node, cred);
+ }
+ };
+ }
+ }
+
+ /** */
+ private static class AuthenticationEvent {
+ /** */
+ UUID clusterNodeId;
+
+ /** */
+ Certificate[] certs;
+
+ /** */
+ AuthenticationEvent(UUID clusterNodeId, Certificate[] certs) {
+ this.clusterNodeId = clusterNodeId;
+ this.certs = certs;
+ }
+ }
+}
diff --git
a/modules/core/src/test/java/org/apache/ignite/testsuites/SecurityTestSuite.java
b/modules/core/src/test/java/org/apache/ignite/testsuites/SecurityTestSuite.java
index da3e8c262ae..52d0c722bb0 100644
---
a/modules/core/src/test/java/org/apache/ignite/testsuites/SecurityTestSuite.java
+++
b/modules/core/src/test/java/org/apache/ignite/testsuites/SecurityTestSuite.java
@@ -19,6 +19,7 @@ package org.apache.ignite.testsuites;
import
org.apache.ignite.internal.processors.security.IgniteSecurityProcessorTest;
import org.apache.ignite.internal.processors.security.InvalidServerTest;
+import
org.apache.ignite.internal.processors.security.NodeConnectionCertificateCapturingTest;
import
org.apache.ignite.internal.processors.security.NodeSecurityContextPropagationTest;
import
org.apache.ignite.internal.processors.security.SecurityContextInternalFuturePropagationTest;
import
org.apache.ignite.internal.processors.security.cache.CacheOperationPermissionCheckTest;
@@ -143,6 +144,7 @@ import org.junit.runners.Suite;
NodeJoinPermissionsTest.class,
ActivationOnJoinWithoutPermissionsWithPersistenceTest.class,
SecurityContextInternalFuturePropagationTest.class,
+ NodeConnectionCertificateCapturingTest.class,
})
public class SecurityTestSuite {
/** */
