This is an automated email from the ASF dual-hosted git repository.
thelabdude pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr-operator.git
The following commit(s) were added to refs/heads/main by this push:
new 3d14af0 Option to watch for updates to the mTLS cert used by the
operator to call Solr pods (#318)
3d14af0 is described below
commit 3d14af0808a3453fb98dd3ff30c033b22e0e7a07
Author: Timothy Potter <[email protected]>
AuthorDate: Wed Sep 1 10:15:25 2021 -0600
Option to watch for updates to the mTLS cert used by the operator to call
Solr pods (#318)
Co-authored-by: Houston Putman <[email protected]>
---
build/NOTICE-ADDITION | 35 +++++++
docs/running-the-operator.md | 5 +-
go.mod | 1 +
helm/solr-operator/Chart.yaml | 7 ++
helm/solr-operator/README.md | 1 +
helm/solr-operator/templates/deployment.yaml | 2 +
helm/solr-operator/values.yaml | 1 +
main.go | 144 ++++++++++++++++++++++-----
8 files changed, 170 insertions(+), 26 deletions(-)
diff --git a/build/NOTICE-ADDITION b/build/NOTICE-ADDITION
index f1ec400..62bd388 100644
--- a/build/NOTICE-ADDITION
+++ b/build/NOTICE-ADDITION
@@ -1,2 +1,37 @@
This project uses the hashicorp/golang-lru project, which is MPL 2.0 licensed.
The source code can be found at
https://github.com/hashicorp/golang-lru
+
+This project uses the runtime dependency https://github.com/fsnotify/fsnotify,
which is BSD-3 licensed.
+See the below notice, provided by the project.
+
+=================================================
+== FSNotify Notice ==
+=================================================
+Copyright (c) 2012 The Go Authors. All rights reserved.
+Copyright (c) 2012-2019 fsnotify Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+ * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/docs/running-the-operator.md b/docs/running-the-operator.md
index 3ada501..41e90e9 100644
--- a/docs/running-the-operator.md
+++ b/docs/running-the-operator.md
@@ -140,4 +140,7 @@ The CA certificate needs to be stored in Kubernetes secret
in PEM format and pro
```
In most cases, you'll also want to configure the operator with
`mTLS.insecureSkipVerify=true` (the default) as you'll want the operator to
skip hostname verification for Solr pods.
-Setting `mTLS.insecureSkipVerify` to `false` means the operator will enforce
hostname verification for the certificate provided by Solr pods.
\ No newline at end of file
+Setting `mTLS.insecureSkipVerify` to `false` means the operator will enforce
hostname verification for the certificate provided by Solr pods.
+
+By default, the operator watches for updates to the mTLS client certificate
(mounted from the `mTLS.clientCertSecret` secret) and then refreshes the HTTP
client to use the updated certificate.
+To disable this behavior, configure the operator using: `--set
mTLS.watchForUpdates=false`.
\ No newline at end of file
diff --git a/go.mod b/go.mod
index 4760908..c6dfc02 100644
--- a/go.mod
+++ b/go.mod
@@ -5,6 +5,7 @@ go 1.16
require (
github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c //
indirect
github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484 //
indirect
+ github.com/fsnotify/fsnotify v1.4.9
github.com/go-logr/logr v0.4.0
github.com/go-logr/zapr v0.2.0 // indirect
github.com/google/gofuzz v1.2.0 // indirect
diff --git a/helm/solr-operator/Chart.yaml b/helm/solr-operator/Chart.yaml
index 860dc5d..c0f80e2 100644
--- a/helm/solr-operator/Chart.yaml
+++ b/helm/solr-operator/Chart.yaml
@@ -156,6 +156,13 @@ annotations:
url: https://github.com/apache/solr-operator/issues/290
- name: Github PR
url: https://github.com/apache/solr-operator/pull/311
+ - kind: added
+ description: Option to watch for updates to the mTLS client certificate
used by the operator to call Solr pods.
+ links:
+ - name: Github Issue
+ url: https://github.com/apache/solr-operator/issues/317
+ - name: Github PR
+ url: https://github.com/apache/solr-operator/pull/318
artifacthub.io/images: |
- name: solr-operator
image: apache/solr-operator:v0.4.0-prerelease
diff --git a/helm/solr-operator/README.md b/helm/solr-operator/README.md
index 841e9a5..50870ca 100644
--- a/helm/solr-operator/README.md
+++ b/helm/solr-operator/README.md
@@ -163,6 +163,7 @@ The command removes all the Kubernetes components
associated with the chart and
| mTLS.caCertSecretKey | string | `""` | Name of a Kubernetes secret, in the
same namespace, that contains PEM encoded Root CA Certificate to use when
connecting to Solr with Client Auth. |
| mTLS.caCertSecret | string | `""` | Name of the key in the `caCertSecret`
that contains the Root CA Cert as a value. |
| mTLS.insecureSkipVerify | boolean | `true` | Skip server certificate and
hostname verification when connecting to Solr with ClientAuth. |
+| mTLS.watchForUpdates | boolean | `true` | Watch for updates to the mTLS
certificate to reload the HTTP client used to call Solr pods with an updated
client certificate. |
### Running the Solr Operator
diff --git a/helm/solr-operator/templates/deployment.yaml
b/helm/solr-operator/templates/deployment.yaml
index e6f2991..946c81c 100644
--- a/helm/solr-operator/templates/deployment.yaml
+++ b/helm/solr-operator/templates/deployment.yaml
@@ -64,6 +64,8 @@ spec:
{{- if .Values.mTLS.insecureSkipVerify }}
- --tls-skip-verify-server={{ .Values.mTLS.insecureSkipVerify }}
{{- end }}
+ - --tls-watch-cert={{ .Values.mTLS.watchForUpdates }}
+
env:
- name: POD_NAMESPACE
valueFrom:
diff --git a/helm/solr-operator/values.yaml b/helm/solr-operator/values.yaml
index d8acfdf..c165b21 100644
--- a/helm/solr-operator/values.yaml
+++ b/helm/solr-operator/values.yaml
@@ -76,3 +76,4 @@ mTLS:
caCertSecret: ""
caCertSecretKey: ca-cert.pem
insecureSkipVerify: true
+ watchForUpdates: true
diff --git a/main.go b/main.go
index a87b695..adc4321 100644
--- a/main.go
+++ b/main.go
@@ -26,9 +26,11 @@ import (
"github.com/apache/solr-operator/controllers"
"github.com/apache/solr-operator/controllers/util/solr_api"
"github.com/apache/solr-operator/version"
+ "github.com/fsnotify/fsnotify"
"io/ioutil"
"net/http"
"os"
+ "path/filepath"
"runtime"
"sigs.k8s.io/controller-runtime/pkg/cache"
"strings"
@@ -65,6 +67,10 @@ var (
clientCertPath string
clientCertKeyPath string
caCertPath string
+ clientCertWatch bool
+
+ // Ref to the active client certificate, will get updated when the
secret changes if watch enabled
+ clientCertificate *tls.Certificate
)
func init() {
@@ -82,6 +88,7 @@ func init() {
flag.StringVar(&clientCertKeyPath, "tls-client-cert-key-path", "",
"Path where a TLS client cert key can be found")
flag.StringVar(&caCertPath, "tls-ca-cert-path", "", "Path where a
Certificate Authority (CA) cert in PEM format can be found")
+ flag.BoolVar(&clientCertWatch, "tls-watch-cert", true, "Controls
whether the operator performs a hot reload of the mTLS when it gets updated;
set to false to disable watching for updates to the TLS cert.")
flag.Parse()
}
@@ -147,8 +154,21 @@ func main() {
controllers.UseZkCRD(useZookeeperCRD)
- if err = initMTLSConfig(); err != nil {
- os.Exit(1)
+ // watch TLS files for update
+ if clientCertPath != "" {
+ var watcher *fsnotify.Watcher
+ if clientCertWatch {
+ watcher, err = fsnotify.NewWatcher()
+ if err != nil {
+ setupLog.Error(err, "Create new file watcher
failed")
+ os.Exit(1)
+ }
+ defer watcher.Close()
+ }
+
+ if err = initMTLSConfig(watcher); err != nil {
+ os.Exit(1)
+ }
}
if err = (&controllers.SolrCloudReconciler{
@@ -181,36 +201,110 @@ func main() {
}
}
-func initMTLSConfig() error {
- if clientCertPath != "" {
- setupLog.Info("mTLS config", "clientSkipVerify",
clientSkipVerify, "clientCertPath", clientCertPath,
- "clientCertKeyPath", clientCertKeyPath, "caCertPath",
caCertPath)
+// Setup for mTLS with Solr pods with hot reload support using the fsnotify
Watcher
+func initMTLSConfig(watcher *fsnotify.Watcher) error {
+ setupLog.Info("mTLS config", "clientSkipVerify", clientSkipVerify,
"clientCertPath", clientCertPath,
+ "clientCertKeyPath", clientCertKeyPath, "caCertPath",
caCertPath, "clientCertWatch", clientCertWatch)
- // Load client cert information from files
- clientCert, err := tls.LoadX509KeyPair(clientCertPath,
clientCertKeyPath)
- if err != nil {
- setupLog.Error(err, "Error loading clientCert pair for
mTLS transport", "certPath", clientCertPath, "keyPath", clientCertKeyPath)
- return err
+ // Load client cert information from files
+ clientCert, err := tls.LoadX509KeyPair(clientCertPath,
clientCertKeyPath)
+ if err != nil {
+ setupLog.Error(err, "Error loading clientCert pair for mTLS
transport", "certPath", clientCertPath, "keyPath", clientCertKeyPath)
+ return err
+ }
+
+ if watcher != nil {
+ // If the cert file is a symlink (which is the case when loaded
from a secret), then we need to re-add the watch after the
+ // Right now, it will always be a symlink, so this is just
future proofing when the cert gets loaded from a CSI driver
+ isSymlink := false
+ clientCertFile, _ := filepath.EvalSymlinks(clientCertPath)
+ if clientCertFile != clientCertPath {
+ isSymlink = true
}
- mTLSTransport := http.DefaultTransport.(*http.Transport).Clone()
- mTLSTransport.TLSClientConfig = &tls.Config{Certificates:
[]tls.Certificate{clientCert}, InsecureSkipVerify: clientSkipVerify}
-
- // Add the rootCA if one is provided
- if caCertPath != "" {
- if caCertBytes, err := ioutil.ReadFile(caCertPath); err
== nil {
- caCertPool := x509.NewCertPool()
- caCertPool.AppendCertsFromPEM(caCertBytes)
- mTLSTransport.TLSClientConfig.ClientCAs =
caCertPool
- setupLog.Info("Configured the custom CA pem for
the mTLS transport", "path", caCertPath)
- } else {
- setupLog.Error(err, "Cannot read provided CA
pem for mTLS transport", "path", caCertPath)
- return err
+ // Watch cert files for updates
+ go func() {
+ for {
+ select {
+ case event, ok := <-watcher.Events:
+ if !ok {
+ return
+ }
+ // If the cert was loaded from a
secret, then the path will be for a symlink and an update comes in as a REMOVE
event
+ // otherwise, look for a write to the
real file
+ if ((isSymlink &&
event.Op&fsnotify.Remove == fsnotify.Remove) || (event.Op&fsnotify.Write ==
fsnotify.Write)) && event.Name == clientCertPath {
+ clientCertFile, _ :=
filepath.EvalSymlinks(clientCertPath)
+ setupLog.Info("mTLS cert file
updated", "certPath", clientCertPath, "certFile", clientCertFile)
+
+ clientCert, err =
tls.LoadX509KeyPair(clientCertPath, clientCertKeyPath)
+ if err == nil {
+ // update our global
client certificate to the new one
+ clientCertificate =
&clientCert
+ setupLog.Info("Updated
mTLS Http Client after update to cert", "certPath", clientCertPath)
+ } else {
+ // will keep using the
old cert, which eventually will cause failures when the old cert expires
+ setupLog.Error(err,
"Error loading clientCert pair (after update) for mTLS transport", "certPath",
clientCertPath, "keyPath", clientCertKeyPath)
+ }
+
+ // If the symlink we were
watching was removed, re-add the watch
+ if event.Op&fsnotify.Remove ==
fsnotify.Remove {
+ err =
watcher.Add(clientCertPath)
+ if err != nil {
+
setupLog.Error(err, "Re-add fsnotify watch for cert failed", "certPath",
clientCertPath)
+ } else {
+
setupLog.Info("Re-added watch for symlink to mTLS cert file", "certPath",
clientCertPath, "certFile", clientCertFile)
+ }
+ }
+ }
+ case err, ok := <-watcher.Errors:
+ if !ok {
+ return
+ }
+ setupLog.Error(err, "fsnotify error")
+ }
}
+ }()
+
+ err = watcher.Add(clientCertPath)
+ if err != nil {
+ setupLog.Error(err, "Add fsnotify watch for cert
failed", "certPath", clientCertPath)
+ return err
}
+ setupLog.Info("Added fsnotify watch for mTLS cert file",
"certPath", clientCertPath, "certFile", clientCertFile, "isSymlink", isSymlink)
+ } else {
+ setupLog.Info("Watch for mTLS cert updates disabled",
"certPath", clientCertPath)
+ }
- solr_api.SetMTLSHttpClient(&http.Client{Transport:
mTLSTransport})
+ clientCertificate = &clientCert
+ mTLSTransport, err := buildTLSTransport()
+ if err != nil {
+ return err
}
+ solr_api.SetMTLSHttpClient(&http.Client{Transport: mTLSTransport})
return nil
}
+
+func buildTLSTransport() (*http.Transport, error) {
+ mTLSTransport := http.DefaultTransport.(*http.Transport).Clone()
+ mTLSTransport.TLSClientConfig = &tls.Config{GetClientCertificate:
getClientCertificate, InsecureSkipVerify: clientSkipVerify}
+
+ // Add the rootCA if one is provided
+ if caCertPath != "" {
+ if caCertBytes, err := ioutil.ReadFile(caCertPath); err == nil {
+ caCertPool := x509.NewCertPool()
+ caCertPool.AppendCertsFromPEM(caCertBytes)
+ mTLSTransport.TLSClientConfig.ClientCAs = caCertPool
+ setupLog.Info("Configured the custom CA pem for the
mTLS transport", "path", caCertPath)
+ } else {
+ setupLog.Error(err, "Cannot read provided CA pem for
mTLS transport", "path", caCertPath)
+ return nil, err
+ }
+ }
+
+ return mTLSTransport, nil
+}
+
+func getClientCertificate(*tls.CertificateRequestInfo) (*tls.Certificate,
error) {
+ return clientCertificate, nil
+}
