Repository: cassandra
Updated Branches:
  refs/heads/trunk 714703a08 -> 473e8dfd7


Add hot reloading of SSL Certificates

patch by Dinesh Joshi; reviewed by jasobrown for CASSANDRA-14222


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/473e8dfd
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/473e8dfd
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/473e8dfd

Branch: refs/heads/trunk
Commit: 473e8dfd7be95815ee10502f021bd7deb8734fba
Parents: 714703a
Author: Dinesh Joshi <dinesh.jo...@yahoo.com>
Authored: Wed Feb 7 15:08:01 2018 -0800
Committer: Jason Brown <jasedbr...@gmail.com>
Committed: Fri Feb 9 05:32:21 2018 -0800

----------------------------------------------------------------------
 doc/source/operating/security.rst               |  10 ++
 .../cassandra/config/DatabaseDescriptor.java    |   8 +
 .../apache/cassandra/net/MessagingService.java  |   7 +
 .../cassandra/net/MessagingServiceMBean.java    |   2 +
 .../apache/cassandra/security/SSLFactory.java   | 148 ++++++++++++++++++-
 .../org/apache/cassandra/tools/NodeProbe.java   |   5 +
 .../org/apache/cassandra/tools/NodeTool.java    |   3 +-
 .../cassandra/tools/ReloadSslCertificates.java  |  30 ++++
 .../cassandra/security/SSLFactoryTest.java      |  38 ++++-
 9 files changed, 239 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/473e8dfd/doc/source/operating/security.rst
----------------------------------------------------------------------
diff --git a/doc/source/operating/security.rst 
b/doc/source/operating/security.rst
index 21245fd..1859dbc 100644
--- a/doc/source/operating/security.rst
+++ b/doc/source/operating/security.rst
@@ -56,6 +56,16 @@ for more details.
 For information on generating the keystore and truststore files used in SSL 
communications, see the
 `java documentation on creating keystores 
<http://download.oracle.com/javase/6/docs/technotes/guides/security/jsse/JSSERefGuide.html#CreateKeystore>`__
 
+SSL Certificate Hot Reloading
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Beginning with Cassandra 4, Cassandra supports hot reloading of SSL 
Certificates. If SSL/TLS support is enabled in Cassandra,
+the node periodically polls the Trust and Key Stores specified in 
cassandra.yaml. When the files are updated, Cassandra will
+reload them and use them for subsequent connections. Please note that the 
Trust & Key Store passwords are part of the yaml so
+the updated files should also use the same passwords. The default polling 
interval is 10 minutes.
+
+Certificate Hot reloading may also be triggered using the ``nodetool 
reloadssl`` command. Use this if you want to Cassandra to
+immediately notice the changed certificates.
+
 Inter-node Encryption
 ~~~~~~~~~~~~~~~~~~~~~
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/473e8dfd/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java 
b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index 8e831cf..0714245 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -60,6 +60,7 @@ import org.apache.cassandra.locator.SeedProvider;
 import org.apache.cassandra.net.BackPressureStrategy;
 import org.apache.cassandra.net.RateBasedBackPressure;
 import org.apache.cassandra.security.EncryptionContext;
+import org.apache.cassandra.security.SSLFactory;
 import org.apache.cassandra.service.CacheService.CacheType;
 import org.apache.cassandra.utils.FBUtilities;
 
@@ -322,6 +323,8 @@ public class DatabaseDescriptor
         applySeedProvider();
 
         applyEncryptionContext();
+
+        applySslContextHotReload();
     }
 
     private static void applySimpleConfig()
@@ -865,6 +868,11 @@ public class DatabaseDescriptor
         encryptionContext = new 
EncryptionContext(conf.transparent_data_encryption_options);
     }
 
+    public static void applySslContextHotReload()
+    {
+        SSLFactory.initHotReloading(conf.server_encryption_options, 
conf.client_encryption_options, false);
+    }
+
     public static void applySeedProvider()
     {
         // load the seeds for node contact points

http://git-wip-us.apache.org/repos/asf/cassandra/blob/473e8dfd/src/java/org/apache/cassandra/net/MessagingService.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/net/MessagingService.java 
b/src/java/org/apache/cassandra/net/MessagingService.java
index 9f00d27..8fdb395 100644
--- a/src/java/org/apache/cassandra/net/MessagingService.java
+++ b/src/java/org/apache/cassandra/net/MessagingService.java
@@ -101,6 +101,7 @@ import 
org.apache.cassandra.net.async.NettyFactory.InboundInitializer;
 import org.apache.cassandra.repair.messages.RepairMessage;
 import org.apache.cassandra.schema.MigrationManager;
 import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.security.SSLFactory;
 import org.apache.cassandra.service.AbstractWriteResponseHandler;
 import org.apache.cassandra.service.StorageProxy;
 import org.apache.cassandra.service.StorageService;
@@ -1664,4 +1665,10 @@ public final class MessagingService implements 
MessagingServiceMBean
         }
         return true;
     }
+
+    @Override
+    public void reloadSslCertificates()
+    {
+        SSLFactory.checkCertFilesForHotReloading();
+    }
 }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/473e8dfd/src/java/org/apache/cassandra/net/MessagingServiceMBean.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/net/MessagingServiceMBean.java 
b/src/java/org/apache/cassandra/net/MessagingServiceMBean.java
index f4a0c43..6adb891 100644
--- a/src/java/org/apache/cassandra/net/MessagingServiceMBean.java
+++ b/src/java/org/apache/cassandra/net/MessagingServiceMBean.java
@@ -129,4 +129,6 @@ public interface MessagingServiceMBean
     public boolean isBackPressureEnabled();
 
     public int getVersion(String address) throws UnknownHostException;
+
+    void reloadSslCertificates();
 }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/473e8dfd/src/java/org/apache/cassandra/security/SSLFactory.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/security/SSLFactory.java 
b/src/java/org/apache/cassandra/security/SSLFactory.java
index a931f5f..0bf769c 100644
--- a/src/java/org/apache/cassandra/security/SSLFactory.java
+++ b/src/java/org/apache/cassandra/security/SSLFactory.java
@@ -18,16 +18,20 @@
 package org.apache.cassandra.security;
 
 
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.security.KeyStore;
 import java.security.cert.X509Certificate;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Date;
 import java.util.Enumeration;
 import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 import javax.net.ssl.KeyManagerFactory;
 import javax.net.ssl.SSLContext;
@@ -37,6 +41,7 @@ import javax.net.ssl.TrustManagerFactory;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Sets;
@@ -49,7 +54,7 @@ import io.netty.handler.ssl.SslContext;
 import io.netty.handler.ssl.SslContextBuilder;
 import io.netty.handler.ssl.SslProvider;
 import io.netty.handler.ssl.SupportedCipherSuiteFilter;
-import io.netty.util.ReferenceCountUtil;
+import org.apache.cassandra.concurrent.ScheduledExecutors;
 import org.apache.cassandra.config.EncryptionOptions;
 
 /**
@@ -78,6 +83,67 @@ public final class SSLFactory
     private static final AtomicReference<SslContext> serverSslContext = new 
AtomicReference<>();
 
     /**
+     * List of files that trigger hot reloading of SSL certificates
+     */
+    private static volatile List<HotReloadableFile> hotReloadableFiles = 
ImmutableList.of();
+
+    /**
+     * Default initial delay for hot reloading
+     */
+    public static final int DEFAULT_HOT_RELOAD_INITIAL_DELAY_SEC = 600;
+
+    /**
+     * Default periodic check delay for hot reloading
+     */
+    public static final int DEFAULT_HOT_RELOAD_PERIOD_SEC = 600;
+
+    /**
+     * State variable to maintain initialization invariant
+     */
+    private static boolean isHotReloadingInitialized = false;
+
+    /**
+     * Helper class for hot reloading SSL Contexts
+     */
+    private static class HotReloadableFile
+    {
+        enum Type
+        {
+            SERVER,
+            CLIENT
+        }
+
+        private final File file;
+        private volatile long lastModTime;
+        private final Type certType;
+
+        HotReloadableFile(String path, Type type)
+        {
+            file = new File(path);
+            lastModTime = file.lastModified();
+            certType = type;
+        }
+
+        boolean shouldReload()
+        {
+            long curModTime = file.lastModified();
+            boolean result = curModTime != lastModTime;
+            lastModTime = curModTime;
+            return result;
+        }
+
+        public boolean isServer()
+        {
+            return certType == Type.SERVER;
+        }
+
+        public boolean isClient()
+        {
+            return certType == Type.CLIENT;
+        }
+    }
+
+    /**
      * Create a JSSE {@link SSLContext}.
      */
     @SuppressWarnings("resource")
@@ -176,10 +242,14 @@ public final class SSLFactory
     @VisibleForTesting
     static SslContext getSslContext(EncryptionOptions options, boolean 
buildTruststore, boolean forServer, boolean useOpenSsl) throws IOException
     {
-        if (forServer && serverSslContext.get() != null)
-            return serverSslContext.get();
-        if (!forServer && clientSslContext.get() != null)
-            return clientSslContext.get();
+
+        SslContext sslContext;
+
+        if (forServer && (sslContext = serverSslContext.get()) != null)
+            return sslContext;
+
+        if (!forServer && (sslContext = clientSslContext.get()) != null)
+            return sslContext;
 
         /*
             There is a case where the netty/openssl combo might not support 
using KeyManagerFactory. specifically,
@@ -219,7 +289,73 @@ public final class SSLFactory
         if (ref.compareAndSet(null, ctx))
             return ctx;
 
-        ReferenceCountUtil.release(ctx);
         return ref.get();
     }
+
+    /**
+     * Performs a lightweight check whether the certificate files have been 
refreshed.
+     *
+     * @throws IllegalStateException if {@link 
#initHotReloading(EncryptionOptions.ServerEncryptionOptions, EncryptionOptions, 
boolean)}
+     * is not called first
+     */
+    public static void checkCertFilesForHotReloading()
+    {
+        if (!isHotReloadingInitialized)
+            throw new IllegalStateException("Hot reloading functionality has 
not been initialized.");
+
+        logger.trace("Checking whether certificates have been updated");
+
+        if (hotReloadableFiles.stream().anyMatch(f -> f.isServer() && 
f.shouldReload()))
+        {
+            logger.info("Server ssl certificates have been updated. Reseting 
the context for new peer connections.");
+            serverSslContext.set(null);
+        }
+
+        if (hotReloadableFiles.stream().anyMatch(f -> f.isClient() && 
f.shouldReload()))
+        {
+            logger.info("Client ssl certificates have been updated. Reseting 
the context for new client connections.");
+            clientSslContext.set(null);
+        }
+    }
+
+    /**
+     * Determines whether to hot reload certificates and schedules a periodic 
task for it.
+     *
+     * @param serverEncryptionOptions
+     * @param clientEncryptionOptions
+     */
+    public static synchronized void 
initHotReloading(EncryptionOptions.ServerEncryptionOptions 
serverEncryptionOptions,
+                                                     EncryptionOptions 
clientEncryptionOptions,
+                                                     boolean force)
+    {
+        if (isHotReloadingInitialized && !force)
+            return;
+
+        logger.debug("Initializing hot reloading SSLContext");
+
+        List<HotReloadableFile> fileList = new ArrayList<>();
+
+        if (serverEncryptionOptions.enabled)
+        {
+            fileList.add(new 
HotReloadableFile(serverEncryptionOptions.keystore, 
HotReloadableFile.Type.SERVER));
+            fileList.add(new 
HotReloadableFile(serverEncryptionOptions.truststore, 
HotReloadableFile.Type.SERVER));
+        }
+
+        if (clientEncryptionOptions.enabled)
+        {
+            fileList.add(new 
HotReloadableFile(clientEncryptionOptions.keystore, 
HotReloadableFile.Type.CLIENT));
+            fileList.add(new 
HotReloadableFile(clientEncryptionOptions.truststore, 
HotReloadableFile.Type.CLIENT));
+        }
+
+        hotReloadableFiles = ImmutableList.copyOf(fileList);
+
+        if (!isHotReloadingInitialized)
+        {
+            
ScheduledExecutors.scheduledTasks.scheduleWithFixedDelay(SSLFactory::checkCertFilesForHotReloading,
+                                                                     
DEFAULT_HOT_RELOAD_INITIAL_DELAY_SEC,
+                                                                     
DEFAULT_HOT_RELOAD_PERIOD_SEC, TimeUnit.SECONDS);
+        }
+
+        isHotReloadingInitialized = true;
+    }
 }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/473e8dfd/src/java/org/apache/cassandra/tools/NodeProbe.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java 
b/src/java/org/apache/cassandra/tools/NodeProbe.java
index 69b64ab..7ce5341 100644
--- a/src/java/org/apache/cassandra/tools/NodeProbe.java
+++ b/src/java/org/apache/cassandra/tools/NodeProbe.java
@@ -1647,6 +1647,11 @@ public class NodeProbe implements AutoCloseable
     {
         return arsProxy;
     }
+
+    public void reloadSslCerts()
+    {
+        msProxy.reloadSslCertificates();
+    }
 }
 
 class ColumnFamilyStoreMBeanIterator implements Iterator<Map.Entry<String, 
ColumnFamilyStoreMBean>>

http://git-wip-us.apache.org/repos/asf/cassandra/blob/473e8dfd/src/java/org/apache/cassandra/tools/NodeTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/NodeTool.java 
b/src/java/org/apache/cassandra/tools/NodeTool.java
index 81f2023..2b6fabf 100644
--- a/src/java/org/apache/cassandra/tools/NodeTool.java
+++ b/src/java/org/apache/cassandra/tools/NodeTool.java
@@ -156,7 +156,8 @@ public class NodeTool
                 RefreshSizeEstimates.class,
                 RelocateSSTables.class,
                 ViewBuildStatus.class,
-                HandoffWindow.class
+                HandoffWindow.class,
+                ReloadSslCertificates.class
         );
 
         Cli.CliBuilder<Runnable> builder = Cli.builder("nodetool");

http://git-wip-us.apache.org/repos/asf/cassandra/blob/473e8dfd/src/java/org/apache/cassandra/tools/ReloadSslCertificates.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/ReloadSslCertificates.java 
b/src/java/org/apache/cassandra/tools/ReloadSslCertificates.java
new file mode 100644
index 0000000..f38b8c0
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/ReloadSslCertificates.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.tools;
+
+import io.airlift.airline.Command;
+
+@Command(name = "reloadssl", description = "Signals Cassandra to reload SSL 
certificates")
+public class ReloadSslCertificates extends NodeTool.NodeToolCmd
+{
+    @Override
+    public void execute(NodeProbe probe)
+    {
+        probe.reloadSslCerts();
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/cassandra/blob/473e8dfd/test/unit/org/apache/cassandra/security/SSLFactoryTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/security/SSLFactoryTest.java 
b/test/unit/org/apache/cassandra/security/SSLFactoryTest.java
index 61933a5..5153a11 100644
--- a/test/unit/org/apache/cassandra/security/SSLFactoryTest.java
+++ b/test/unit/org/apache/cassandra/security/SSLFactoryTest.java
@@ -18,16 +18,12 @@
 */
 package org.apache.cassandra.security;
 
+import java.io.File;
 import java.io.IOException;
-import java.net.InetAddress;
 import java.security.cert.CertificateException;
 import java.util.Arrays;
-import javax.net.ssl.SSLServerSocket;
 import javax.net.ssl.TrustManagerFactory;
 
-import com.google.common.base.Predicates;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
@@ -162,4 +158,36 @@ public class SSLFactoryTest
         SSLFactory.buildKeyManagerFactory(options);
         Assert.assertTrue(SSLFactory.checkedExpiry);
     }
+
+    @Test
+    public void testSslContextReload_HappyPath() throws IOException, 
InterruptedException
+    {
+        try
+        {
+            EncryptionOptions options = addKeystoreOptions(encryptionOptions);
+            options.enabled = true;
+
+            SSLFactory.initHotReloading((ServerEncryptionOptions) options, 
options, true);
+
+            SslContext oldCtx = SSLFactory.getSslContext(options, true, true, 
OpenSsl.isAvailable());
+            File keystoreFile = new File(options.keystore);
+
+            SSLFactory.checkCertFilesForHotReloading();
+            Thread.sleep(5000);
+            keystoreFile.setLastModified(System.currentTimeMillis());
+
+            SSLFactory.checkCertFilesForHotReloading();
+            SslContext newCtx = SSLFactory.getSslContext(options, true, true, 
OpenSsl.isAvailable());
+
+            Assert.assertNotSame(oldCtx, newCtx);
+        }
+        catch (Exception e)
+        {
+            throw e;
+        }
+        finally
+        {
+            DatabaseDescriptor.loadConfig();
+        }
+    }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org

Reply via email to