This is an automated email from the ASF dual-hosted git repository.
gerlowskija pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr-operator.git
The following commit(s) were added to refs/heads/main by this push:
new 7beea83 Use Solr's /admin/info/health for pod readiness checks (#629)
7beea83 is described below
commit 7beea838546176ae144bf1c3c60a890bbc2332b0
Author: Jason Gerlowski <[email protected]>
AuthorDate: Thu Oct 5 10:47:45 2023 -0400
Use Solr's /admin/info/health for pod readiness checks (#629)
Now that the operator expects Solr versions >=8.11, we can use Solr's
/admin/info/health endpoint as our default readiness probe.
(/admin/info/system remains in use for liveness and startup probes.)
---
.../solrcloud_controller_basic_auth_test.go | 28 +++++++++++++++-------
controllers/solrcloud_controller_tls_test.go | 10 ++++----
controllers/util/solr_security_util.go | 11 +++++----
controllers/util/solr_util.go | 26 ++++++++++++--------
docs/solr-cloud/solr-cloud-crd.md | 26 ++++++++++++--------
docs/upgrade-notes.md | 6 ++++-
hack/release/smoke_test/test_cluster.sh | 2 +-
helm/solr/Chart.yaml | 7 ++++++
8 files changed, 76 insertions(+), 40 deletions(-)
diff --git a/controllers/solrcloud_controller_basic_auth_test.go
b/controllers/solrcloud_controller_basic_auth_test.go
index 7a4cee8..a91846f 100644
--- a/controllers/solrcloud_controller_basic_auth_test.go
+++ b/controllers/solrcloud_controller_basic_auth_test.go
@@ -208,13 +208,17 @@ var boostrapedSecretKeys = []string{
func expectStatefulSetBasicAuthConfig(ctx context.Context, sc
*solrv1beta1.SolrCloud, expectBootstrapSecret bool) *appsv1.StatefulSet {
Expect(sc.Spec.SolrSecurity).To(Not(BeNil()), "solrSecurity is not
configured for this SolrCloud instance!")
- expProbePath := "/solr/admin/info/system"
+ expLivenessProbePath := "/solr/admin/info/system"
+ expReadinessProbePath := "/solr/admin/info/health"
if sc.Spec.CustomSolrKubeOptions.PodOptions != nil &&
sc.Spec.CustomSolrKubeOptions.PodOptions.LivenessProbe != nil {
- expProbePath =
sc.Spec.CustomSolrKubeOptions.PodOptions.LivenessProbe.HTTPGet.Path
+ expLivenessProbePath =
sc.Spec.CustomSolrKubeOptions.PodOptions.LivenessProbe.HTTPGet.Path
+ }
+ if sc.Spec.CustomSolrKubeOptions.PodOptions != nil &&
sc.Spec.CustomSolrKubeOptions.PodOptions.ReadinessProbe != nil {
+ expReadinessProbePath =
sc.Spec.CustomSolrKubeOptions.PodOptions.ReadinessProbe.HTTPGet.Path
}
stateful := expectStatefulSetWithChecks(ctx, sc, sc.StatefulSetName(),
func(g Gomega, found *appsv1.StatefulSet) {
- expectBasicAuthConfigOnPodTemplateWithGomega(g, sc,
&found.Spec.Template, expectBootstrapSecret, expProbePath)
+ expectBasicAuthConfigOnPodTemplateWithGomega(g, sc,
&found.Spec.Template, expectBootstrapSecret, expLivenessProbePath,
expReadinessProbePath)
})
expectSecretWithChecks(ctx, sc, sc.BasicAuthSecretName(), func(innerG
Gomega, found *corev1.Secret) {
@@ -237,7 +241,9 @@ func expectStatefulSetBasicAuthConfig(ctx context.Context,
sc *solrv1beta1.SolrC
probePaths := util.GetCustomProbePaths(sc)
if len(probePaths) > 0 {
securityJson :=
string(bootstrapSecret.Data["security.json"])
-
Expect(securityJson).To(ContainSubstring(util.DefaultProbePath), "bootstrapped
security.json should have an authz rule for probe path: %s",
util.DefaultProbePath)
+
Expect(securityJson).To(ContainSubstring(util.DefaultLivenessProbePath),
"bootstrapped security.json should have an authz rule for liveness probe path:
%s", util.DefaultLivenessProbePath)
+
Expect(securityJson).To(ContainSubstring(util.DefaultReadinessProbePath),
"bootstrapped security.json should have an authz rule for readiness probe path:
%s", util.DefaultReadinessProbePath)
+
for _, p := range probePaths {
p = p[len("/solr"):] // drop the /solr
part on the path
Expect(securityJson).To(ContainSubstring(p), "bootstrapped security.json should
have an authz rule for probe path: %s", p)
@@ -250,7 +256,7 @@ func expectStatefulSetBasicAuthConfig(ctx context.Context,
sc *solrv1beta1.SolrC
}
// Ensures config is setup for basic-auth enabled Solr pods
-func expectBasicAuthConfigOnPodTemplateWithGomega(g Gomega, solrCloud
*solrv1beta1.SolrCloud, podTemplate *corev1.PodTemplateSpec,
expectBootstrapSecret bool, expProbePath string) *corev1.Container {
+func expectBasicAuthConfigOnPodTemplateWithGomega(g Gomega, solrCloud
*solrv1beta1.SolrCloud, podTemplate *corev1.PodTemplateSpec,
expectBootstrapSecret bool, expLivenessProbePath string, expReadinessProbePath
string) *corev1.Container {
// check the env vars needed for the probes to work with auth
g.Expect(podTemplate.Spec.Containers).To(Not(BeEmpty()), "Solr Pod
requires containers")
mainContainer := podTemplate.Spec.Containers[0]
@@ -282,18 +288,22 @@ func expectBasicAuthConfigOnPodTemplateWithGomega(g
Gomega, solrCloud *solrv1bet
g.Expect(basicAuthSecretVolMount).To(Not(BeNil()), "No Basic
Auth volume mount used in Solr container")
g.Expect(basicAuthSecretVolMount.MountPath).To(Equal("/etc/secrets/"+secretName),
"Wrong path used to mount Basic Auth volume")
- expProbeCmd :=
fmt.Sprintf("JAVA_TOOL_OPTIONS=\"-Dbasicauth=$(cat
/etc/secrets/%s-solrcloud-basic-auth/username):$(cat
/etc/secrets/%s-solrcloud-basic-auth/password)
-Dsolr.httpclient.builder.factory=org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory\"
"+
+ expLivenessProbeCmd :=
fmt.Sprintf("JAVA_TOOL_OPTIONS=\"-Dbasicauth=$(cat
/etc/secrets/%s-solrcloud-basic-auth/username):$(cat
/etc/secrets/%s-solrcloud-basic-auth/password)
-Dsolr.httpclient.builder.factory=org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory\"
"+
"solr api -get \"http://${SOLR_HOST}:8983%s\"";,
- solrCloud.Name, solrCloud.Name, expProbePath)
+ solrCloud.Name, solrCloud.Name, expLivenessProbePath)
+ expReadinessProbeCmd :=
fmt.Sprintf("JAVA_TOOL_OPTIONS=\"-Dbasicauth=$(cat
/etc/secrets/%s-solrcloud-basic-auth/username):$(cat
/etc/secrets/%s-solrcloud-basic-auth/password)
-Dsolr.httpclient.builder.factory=org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory\"
"+
+ "solr api -get \"http://${SOLR_HOST}:8983%s\"";,
+ solrCloud.Name, solrCloud.Name, expReadinessProbePath)
+
g.Expect(mainContainer.LivenessProbe).To(Not(BeNil()), "main
container should have a liveness probe defined")
g.Expect(mainContainer.LivenessProbe.Exec).To(Not(BeNil()),
"liveness probe should have an exec when auth is enabled")
g.Expect(mainContainer.LivenessProbe.Exec.Command).To(Not(BeEmpty()), "liveness
probe command cannot be empty")
-
g.Expect(mainContainer.LivenessProbe.Exec.Command[2]).To(Equal(expProbeCmd),
"liveness probe should invoke java with auth opts")
+
g.Expect(mainContainer.LivenessProbe.Exec.Command[2]).To(Equal(expLivenessProbeCmd),
"liveness probe should invoke java with auth opts")
g.Expect(mainContainer.LivenessProbe.TimeoutSeconds).To(BeEquivalentTo(5),
"liveness probe default timeout should be increased when using basicAuth")
g.Expect(mainContainer.ReadinessProbe).To(Not(BeNil()), "main
container should have a readiness probe defined")
g.Expect(mainContainer.ReadinessProbe.Exec).To(Not(BeNil()),
"readiness probe should have an exec when auth is enabled")
g.Expect(mainContainer.ReadinessProbe.Exec.Command).To(Not(BeEmpty()),
"readiness probe command cannot be empty")
-
g.Expect(mainContainer.ReadinessProbe.Exec.Command[2]).To(Equal(expProbeCmd),
"readiness probe should invoke java with auth opts")
+
g.Expect(mainContainer.ReadinessProbe.Exec.Command[2]).To(Equal(expReadinessProbeCmd),
"readiness probe should invoke java with auth opts")
g.Expect(mainContainer.ReadinessProbe.TimeoutSeconds).To(BeEquivalentTo(5),
"readiness probe default timeout should be increased when using basicAuth")
}
diff --git a/controllers/solrcloud_controller_tls_test.go
b/controllers/solrcloud_controller_tls_test.go
index 22daa96..c24013f 100644
--- a/controllers/solrcloud_controller_tls_test.go
+++ b/controllers/solrcloud_controller_tls_test.go
@@ -124,7 +124,7 @@ var _ = FDescribe("SolrCloud controller - TLS", func() {
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Scheme:
corev1.URISchemeHTTPS,
- Path:
"/solr/admin/info/health",
+ Path:
"/solr/admin/customreadiness",
Port:
intstr.FromInt(8983),
},
},
@@ -134,7 +134,7 @@ var _ = FDescribe("SolrCloud controller - TLS", func() {
solrCloud.Spec.SolrTLS =
createTLSOptions(tlsSecretName, keystorePassKey, false)
})
FIt("has the correct resources", func(ctx context.Context) {
-
Expect(util.GetCustomProbePaths(solrCloud)).To(ConsistOf("/solr/admin/info/health"),
"Utility Probe paths command gives wrong result")
+
Expect(util.GetCustomProbePaths(solrCloud)).To(ConsistOf("/solr/admin/customreadiness"),
"Utility Probe paths command gives wrong result")
verifyUserSuppliedTLSConfig(solrCloud.Spec.SolrTLS,
tlsSecretName, keystorePassKey, tlsSecretName)
By("checking that the User supplied TLS Config is
correct in the generated StatefulSet")
@@ -536,7 +536,7 @@ func expectTLSConfigOnPodTemplateWithGomega(g Gomega,
solrCloud *solrv1beta1.Sol
g.Expect(mainContainer.LivenessProbe.Exec).To(Not(BeNil()),
"liveness probe should have an exec when mTLS is enabled")
g.Expect(mainContainer.LivenessProbe.Exec.Command).To(Not(BeNil()), "liveness
probe should have an exec when mTLS is enabled")
g.Expect(mainContainer.LivenessProbe.Exec.Command).To(HaveLen(3), "liveness
probe command has wrong number of args")
- path := "/solr" + util.DefaultProbePath
+ path := "/solr" + util.DefaultLivenessProbePath
if solrCloud.Spec.CustomSolrKubeOptions.PodOptions != nil &&
solrCloud.Spec.CustomSolrKubeOptions.PodOptions.LivenessProbe != nil {
path =
solrCloud.Spec.CustomSolrKubeOptions.PodOptions.LivenessProbe.HTTPGet.Path
}
@@ -544,7 +544,7 @@ func expectTLSConfigOnPodTemplateWithGomega(g Gomega,
solrCloud *solrv1beta1.Sol
g.Expect(mainContainer.ReadinessProbe).To(Not(BeNil()), "main
container should have a readiness probe defined")
g.Expect(mainContainer.ReadinessProbe.Exec).To(Not(BeNil()),
"readiness probe should have an exec when mTLS is enabled")
g.Expect(mainContainer.ReadinessProbe.Exec.Command).To(HaveLen(3), "readiness
probe command has wrong number of args")
- path = "/solr" + util.DefaultProbePath
+ path = "/solr" + util.DefaultReadinessProbePath
if solrCloud.Spec.CustomSolrKubeOptions.PodOptions != nil &&
solrCloud.Spec.CustomSolrKubeOptions.PodOptions.ReadinessProbe != nil {
path =
solrCloud.Spec.CustomSolrKubeOptions.PodOptions.ReadinessProbe.HTTPGet.Path
}
@@ -553,7 +553,7 @@ func expectTLSConfigOnPodTemplateWithGomega(g Gomega,
solrCloud *solrv1beta1.Sol
g.Expect(mainContainer.StartupProbe).To(Not(BeNil()),
"main container should have a startup probe defined")
g.Expect(mainContainer.StartupProbe.Exec).To(Not(BeNil()), "startup probe
should have an exec when auth is enabled")
g.Expect(mainContainer.StartupProbe.Exec.Command).To(HaveLen(3), "startup probe
command has wrong number of args")
- path = "/solr" + util.DefaultProbePath
+ path = "/solr" + util.DefaultStartupProbePath
if solrCloud.Spec.CustomSolrKubeOptions.PodOptions !=
nil && solrCloud.Spec.CustomSolrKubeOptions.PodOptions.StartupProbe != nil {
path =
solrCloud.Spec.CustomSolrKubeOptions.PodOptions.StartupProbe.HTTPGet.Path
}
diff --git a/controllers/util/solr_security_util.go
b/controllers/util/solr_security_util.go
index 8e76dc0..51caa31 100644
--- a/controllers/util/solr_security_util.go
+++ b/controllers/util/solr_security_util.go
@@ -39,9 +39,11 @@ import (
)
const (
- SecurityJsonFile = "security.json"
- BasicAuthMd5Annotation = "solr.apache.org/basicAuthMd5"
- DefaultProbePath = "/admin/info/system"
+ SecurityJsonFile = "security.json"
+ BasicAuthMd5Annotation = "solr.apache.org/basicAuthMd5"
+ DefaultStartupProbePath = "/admin/info/system"
+ DefaultLivenessProbePath = DefaultStartupProbePath
+ DefaultReadinessProbePath = "/admin/info/health"
)
// Utility struct holding security related config and objects resolved at
runtime needed during reconciliation,
@@ -433,7 +435,8 @@ func solrPasswordHash(passBytes []byte) string {
// Gets a list of probe paths we need to setup authz for
func getProbePaths(solrCloud *solr.SolrCloud) []string {
- probePaths := []string{DefaultProbePath}
+ // Current startup and liveness probes use the same API, so liveness
needn't be explicitly specified here
+ probePaths := []string{DefaultStartupProbePath,
DefaultReadinessProbePath}
probePaths = append(probePaths, GetCustomProbePaths(solrCloud)...)
return uniqueProbePaths(probePaths)
}
diff --git a/controllers/util/solr_util.go b/controllers/util/solr_util.go
index 62a99c8..c6ede52 100644
--- a/controllers/util/solr_util.go
+++ b/controllers/util/solr_util.go
@@ -89,13 +89,9 @@ func GenerateStatefulSet(solrCloud *solr.SolrCloud,
solrCloudStatus *solr.SolrCl
}
defaultProbeTimeout := int32(1)
- defaultHandler := corev1.ProbeHandler{
- HTTPGet: &corev1.HTTPGetAction{
- Scheme: probeScheme,
- Path: "/solr" + DefaultProbePath,
- Port: intstr.FromInt(solrPodPort),
- },
- }
+ defaultStartupProbe := createDefaultProbeHandlerForPath(probeScheme,
solrPodPort, DefaultStartupProbePath)
+ defaultLivenessProbe := createDefaultProbeHandlerForPath(probeScheme,
solrPodPort, DefaultLivenessProbePath)
+ defaultReadinessProbe := createDefaultProbeHandlerForPath(probeScheme,
solrPodPort, DefaultReadinessProbePath)
labels := solrCloud.SharedLabelsWith(solrCloud.GetLabels())
selectorLabels := solrCloud.SharedLabels()
@@ -468,7 +464,7 @@ func GenerateStatefulSet(solrCloud *solr.SolrCloud,
solrCloudStatus *solr.SolrCl
SuccessThreshold: 1,
FailureThreshold: 10,
PeriodSeconds: 5,
- ProbeHandler: defaultHandler,
+ ProbeHandler: defaultStartupProbe,
},
// Kill Solr if it is unavailable for any 60-second
period
LivenessProbe: &corev1.Probe{
@@ -476,7 +472,7 @@ func GenerateStatefulSet(solrCloud *solr.SolrCloud,
solrCloudStatus *solr.SolrCl
SuccessThreshold: 1,
FailureThreshold: 3,
PeriodSeconds: 20,
- ProbeHandler: defaultHandler,
+ ProbeHandler: defaultLivenessProbe,
},
// Do not route requests to solr if it is not available
for any 20-second period
ReadinessProbe: &corev1.Probe{
@@ -484,7 +480,7 @@ func GenerateStatefulSet(solrCloud *solr.SolrCloud,
solrCloudStatus *solr.SolrCl
SuccessThreshold: 1,
FailureThreshold: 2,
PeriodSeconds: 10,
- ProbeHandler: defaultHandler,
+ ProbeHandler: defaultReadinessProbe,
},
VolumeMounts: volumeMounts,
Env: envVars,
@@ -782,6 +778,16 @@ func generateSolrSetupInitContainers(solrCloud
*solr.SolrCloud, solrCloudStatus
return containers
}
+func createDefaultProbeHandlerForPath(probeScheme corev1.URIScheme,
solrPodPort int, path string) corev1.ProbeHandler {
+ return corev1.ProbeHandler{
+ HTTPGet: &corev1.HTTPGetAction{
+ Scheme: probeScheme,
+ Path: "/solr" + path,
+ Port: intstr.FromInt(solrPodPort),
+ },
+ }
+}
+
const DefaultSolrXML = `<?xml version="1.0" encoding="UTF-8" ?>
<solr>
%s
diff --git a/docs/solr-cloud/solr-cloud-crd.md
b/docs/solr-cloud/solr-cloud-crd.md
index b30ea67..8c8874e 100644
--- a/docs/solr-cloud/solr-cloud-crd.md
+++ b/docs/solr-cloud/solr-cloud-crd.md
@@ -930,10 +930,10 @@ The operator configures a command instead of setting the
`Authorization` header
With a command, we can load the username and password from a secret;
Kubernetes will
[update the mounted secret
files](https://kubernetes.io/docs/concepts/configuration/secret/#mounted-secrets-are-updated-automatically)
when the secret changes automatically.
-If you customize the HTTP path for any probes (under
`spec.customSolrKubeOptions.podOptions`),
-then you must use `probesRequireAuth=false` as the operator does not
reconfigure custom HTTP probes to use the command needed to support
`probesRequireAuth=true`.
+By default, the operator will use Solr's `/admin/info/system` endpoint for
startup and liveness probes, and the `/admin/info/health` endpoint for
readiness probes.
+This default can be customized by changing the HTTP path for any probes (under
`spec.customSolrKubeOptions.podOptions`), however this also requires users to
set `probesRequireAuth=false` as the operator does not reconfigure custom HTTP
probes to use the command needed to support `probesRequireAuth=true`.
-If you're running Solr 8+, then we recommend using the `/admin/info/health`
endpoint for your probes using the following config:
+Custom readiness and liveness probes can be specified using configuration like
the following:
```yaml
spec:
...
@@ -942,21 +942,28 @@ spec:
livenessProbe:
httpGet:
scheme: HTTP
- path: /solr/admin/info/health
+ path: /solr/admin/customliveness
port: 8983
readinessProbe:
httpGet:
scheme: HTTP
- path: /solr/admin/info/health
+ path: /solr/admin/customreadiness
port: 8983
```
-Consequently, the bootstrapped `security.json` will include an additional rule
to allow access to the `/admin/info/health` endpoint:
+
+Consequently, the bootstrapped `security.json` will include additional rules
to allow access to the endpoints used by the startup, liveness, and readiness
probes:
```json
{
"name": "k8s-probe-1",
"role": null,
"collection": null,
- "path": "/admin/info/health"
+ "path": "/admin/customliveness"
+ },
+ {
+ "name": "k8s-probe-2",
+ "role": null,
+ "collection": null,
+ "path": "/admin/customreadiness"
}
```
@@ -993,7 +1000,7 @@ Take a moment to review these authorization rules so that
you're aware of the ro
"name": "k8s-probe-0",
"role": null,
"collection": null,
- "path": "/admin/info/system"
+ "path": "/admin/info/health"
},
{
"name": "k8s-status",
@@ -1050,7 +1057,7 @@ A few aspects of the default `security.json`
configuration warrant a closer look
"name": "k8s-probe-0",
"role": null,
"collection": null,
- "path": "/admin/info/system"
+ "path": "/admin/info/health"
}
```
In this case, the `"role":null` indicates this endpoint allows anonymous
access by unknown users.
@@ -1146,7 +1153,6 @@ _Note: be sure to use a stronger password for real
deployments_
Users need to ensure their `security.json` contains the user supplied in the
`basicAuthSecret` has read access to the following endpoints:
```
-/admin/info/system
/admin/info/health
/admin/collections
/admin/metrics
diff --git a/docs/upgrade-notes.md b/docs/upgrade-notes.md
index 128be56..3019e48 100644
--- a/docs/upgrade-notes.md
+++ b/docs/upgrade-notes.md
@@ -130,7 +130,11 @@ _Note that the Helm chart version does not contain a `v`
prefix, which the downl
- The `POD_HOSTNAME` envVar in SolrCloud Pods has been deprecated. Use
`POD_NAME` instead.
-- Use of the `hostPort` system property placeholder in custom solr.xml files
has been deprecated. Use `<int
name="hostPort">${solr.port.advertise:80}</int>`, the default value used by
Solr, instead.
+- Use of the `hostPort` system property placeholder in custom solr.xml files
has been deprecated.
+ Use `<int name="hostPort">${solr.port.advertise:80}</int>`, the default
value used by Solr, instead.
+
+- By default `solrcloud` resources will now use `/admin/info/system` and
`/admin/info/health` for liveness and readiness checks, respectively.
+ Administrators that provide custom `security.json` files for their clusters
should either exempt both of these endpoints from authentication entirely, or
configure permissions ensuring the relevant Solr user account can access them
without issue.
### v0.7.0
- **Kubernetes support is now limited to 1.21+.**
diff --git a/hack/release/smoke_test/test_cluster.sh
b/hack/release/smoke_test/test_cluster.sh
index 26d7ed4..27b3aa1 100755
--- a/hack/release/smoke_test/test_cluster.sh
+++ b/hack/release/smoke_test/test_cluster.sh
@@ -195,7 +195,7 @@ kubectl port-forward service/example-solrcloud-common
18983:80 || true &
sleep 2
printf "\nCheck the admin URL to make sure it works\n"
-curl --silent "http://localhost:18983/solr/admin/info/system"; | grep
'"status":0' > /dev/null
+curl --silent "http://localhost:18983/solr/admin/info/health"; | grep
'"status":0' > /dev/null
printf "\nCreating a test collection\n"
curl --silent
"http://localhost:18983/solr/admin/collections?action=CREATE&name=smoke-test&replicationFactor=1&numShards=2";
| grep '"status":0' > /dev/null
diff --git a/helm/solr/Chart.yaml b/helm/solr/Chart.yaml
index f265d4b..5fda1c1 100644
--- a/helm/solr/Chart.yaml
+++ b/helm/solr/Chart.yaml
@@ -62,6 +62,13 @@ annotations:
url: https://github.com/apache/solr-operator/issues/635
- name: Github PR
url: https://github.com/apache/solr-operator/pull/636
+ - kind: changed
+ description: Solr's `/admin/info/health` API is now used for readiness
probes by default.
+ links:
+ - name: Github Issue
+ url: https://github.com/apache/solr-operator/issues/628
+ - name: Github PR
+ url: https://github.com/apache/solr-operator/pull/629
artifacthub.io/containsSecurityUpdates: "false"
artifacthub.io/recommendations: |
- url: https://artifacthub.io/packages/helm/apache-solr/solr-operator
