This is an automated email from the ASF dual-hosted git repository.
houston pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr-operator.git
The following commit(s) were added to refs/heads/main by this push:
new c9d556c Ability to customize probes for the PrometheusExporter (#297)
c9d556c is described below
commit c9d556c7c6534a1a92f194813596d4ce05e88ad2
Author: Houston Putman <[email protected]>
AuthorDate: Mon Jul 26 10:03:12 2021 -0400
Ability to customize probes for the PrometheusExporter (#297)
---
controllers/controller_utils_test.go | 7 +-
controllers/solrcloud_controller_test.go | 5 +-
.../solrprometheusexporter_controller_test.go | 27 ++++++
controllers/suite_test.go | 9 ++
controllers/util/common.go | 29 +++++++
controllers/util/prometheus_exporter_util.go | 25 ++++--
controllers/util/solr_util.go | 97 ++++++----------------
helm/solr-operator/Chart.yaml | 7 ++
8 files changed, 121 insertions(+), 85 deletions(-)
diff --git a/controllers/controller_utils_test.go
b/controllers/controller_utils_test.go
index ba48ec2..dace14f 100644
--- a/controllers/controller_utils_test.go
+++ b/controllers/controller_utils_test.go
@@ -23,7 +23,6 @@ import (
"github.com/apache/solr-operator/controllers/util"
zkv1beta1
"github.com/pravega/zookeeper-operator/pkg/apis/zookeeper/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "reflect"
"strings"
"testing"
@@ -392,11 +391,11 @@ func testGenericPodEnvVariables(t *testing.T,
expectedEnvVars map[string]string,
}
func testPodTolerations(t *testing.T, expectedTolerations []corev1.Toleration,
foundTolerations []corev1.Toleration) {
- assert.True(t, reflect.DeepEqual(expectedTolerations,
foundTolerations), "Expected tolerations and found tolerations don't match")
+ assert.EqualValues(t, expectedTolerations, foundTolerations, "Expected
tolerations and found tolerations don't match")
}
-func testPodProbe(t *testing.T, expectedProbe *corev1.Probe, foundProbe
*corev1.Probe) {
- assert.True(t, reflect.DeepEqual(expectedProbe, foundProbe), "Expected
probe and found probe don't match")
+func testPodProbe(t *testing.T, expectedProbe *corev1.Probe, foundProbe
*corev1.Probe, probeType string) {
+ assert.EqualValuesf(t, expectedProbe, foundProbe, "Incorrect default
container %s probe", probeType)
}
func testMapsEqual(t *testing.T, mapName string, expected map[string]string,
found map[string]string) {
diff --git a/controllers/solrcloud_controller_test.go
b/controllers/solrcloud_controller_test.go
index 1e9a3e6..5a9ecdb 100644
--- a/controllers/solrcloud_controller_test.go
+++ b/controllers/solrcloud_controller_test.go
@@ -294,8 +294,9 @@ func TestCustomKubeOptionsCloudReconcile(t *testing.T) {
}
testMapsEqual(t, "pod annotations",
util.MergeLabelsOrAnnotations(map[string]string{"solr.apache.org/solrXmlMd5":
fmt.Sprintf("%x", md5.Sum([]byte(configMap.Data["solr.xml"])))},
testPodAnnotations), statefulSet.Spec.Template.Annotations)
testMapsEqual(t, "pod node selectors", testNodeSelectors,
statefulSet.Spec.Template.Spec.NodeSelector)
- testPodProbe(t, testProbeLivenessNonDefaults,
statefulSet.Spec.Template.Spec.Containers[0].LivenessProbe)
- testPodProbe(t, testProbeReadinessNonDefaults,
statefulSet.Spec.Template.Spec.Containers[0].ReadinessProbe)
+ testPodProbe(t, testProbeLivenessNonDefaults,
statefulSet.Spec.Template.Spec.Containers[0].LivenessProbe, "liveness")
+ testPodProbe(t, testProbeReadinessNonDefaults,
statefulSet.Spec.Template.Spec.Containers[0].ReadinessProbe, "readiness")
+ testPodProbe(t, testProbeStartup,
statefulSet.Spec.Template.Spec.Containers[0].StartupProbe, "startup")
assert.Equal(t, []string{"solr", "stop", "-p", "8983"},
statefulSet.Spec.Template.Spec.Containers[0].Lifecycle.PreStop.Exec.Command,
"Incorrect pre-stop command")
testPodTolerations(t, testTolerations,
statefulSet.Spec.Template.Spec.Tolerations)
assert.EqualValues(t, testPriorityClass,
statefulSet.Spec.Template.Spec.PriorityClassName, "Incorrect Priority class
name for Pod Spec")
diff --git a/controllers/solrprometheusexporter_controller_test.go
b/controllers/solrprometheusexporter_controller_test.go
index 271f3c2..be8f96e 100644
--- a/controllers/solrprometheusexporter_controller_test.go
+++ b/controllers/solrprometheusexporter_controller_test.go
@@ -29,6 +29,7 @@ import (
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
+ "k8s.io/apimachinery/pkg/util/intstr"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/manager"
@@ -65,6 +66,8 @@ func TestMetricsReconcileWithoutExporterConfig(t *testing.T) {
InitContainers:
extraContainers1,
ImagePullSecrets:
testAdditionalImagePullSecrets,
TerminationGracePeriodSeconds:
&testTerminationGracePeriodSeconds,
+ LivenessProbe:
testProbeLivenessNonDefaults,
+ ReadinessProbe:
testProbeReadinessNonDefaults,
},
},
ExporterEntrypoint: "/test/entry-point",
@@ -132,6 +135,11 @@ func TestMetricsReconcileWithoutExporterConfig(t
*testing.T) {
assert.ElementsMatch(t, append(testAdditionalImagePullSecrets,
corev1.LocalObjectReference{Name: testImagePullSecretName}),
deployment.Spec.Template.Spec.ImagePullSecrets, "Incorrect imagePullSecrets")
assert.EqualValues(t, &testTerminationGracePeriodSeconds,
deployment.Spec.Template.Spec.TerminationGracePeriodSeconds, "Incorrect
terminationGracePeriodSeconds")
+ testPodProbe(t, testProbeLivenessNonDefaults,
deployment.Spec.Template.Spec.Containers[0].LivenessProbe, "liveness")
+ testPodProbe(t, testProbeReadinessNonDefaults,
deployment.Spec.Template.Spec.Containers[0].ReadinessProbe, "readiness")
+ assert.Nilf(t,
deployment.Spec.Template.Spec.Containers[0].StartupProbe, "%s probe should be
nil since it was not specified", "startup")
+
+ // Check the Service
service := expectService(t, g, requests, expectedMetricsRequest,
metricsSKey, deployment.Spec.Template.Labels)
assert.Equal(t, "true", service.Annotations["prometheus.io/scrape"],
"Metrics Service Prometheus scraping is not enabled.")
assert.EqualValues(t, "solr-metrics", service.Spec.Ports[0].Name,
"Wrong port name on common Service")
@@ -151,6 +159,7 @@ func TestMetricsReconcileWithExporterConfig(t *testing.T) {
Tolerations:
testTolerationsPromExporter,
NodeSelector: testNodeSelectors,
PriorityClassName: testPriorityClass,
+ StartupProbe: testProbeStartup,
},
DeploymentOptions: &solr.DeploymentOptions{
Annotations: testDeploymentAnnotations,
@@ -228,6 +237,24 @@ func TestMetricsReconcileWithExporterConfig(t *testing.T) {
assert.Equal(t, extraVolumes[0].Name,
deployment.Spec.Template.Spec.Volumes[1].Name, "Additional Volume from
podOptions not loaded into pod properly.")
assert.Equal(t, extraVolumes[0].Source,
deployment.Spec.Template.Spec.Volumes[1].VolumeSource, "Additional Volume from
podOptions not loaded into pod properly.")
+ testPodProbe(t, &corev1.Probe{
+ Handler: corev1.Handler{
+ HTTPGet: &corev1.HTTPGetAction{
+ Scheme: corev1.URISchemeHTTP,
+ Path: "/metrics",
+ Port: intstr.FromInt(util.SolrMetricsPort),
+ },
+ },
+ InitialDelaySeconds: 20,
+ TimeoutSeconds: 1,
+ PeriodSeconds: 10,
+ SuccessThreshold: 1,
+ FailureThreshold: 3,
+ }, deployment.Spec.Template.Spec.Containers[0].LivenessProbe,
"liveness")
+ testPodProbe(t, testProbeStartup,
deployment.Spec.Template.Spec.Containers[0].StartupProbe, "startup")
+ assert.Nilf(t,
deployment.Spec.Template.Spec.Containers[0].ReadinessProbe, "%s probe should be
nil since it was not specified", "readiness")
+
+ // Check the Service
expectedServiceLabels :=
util.MergeLabelsOrAnnotations(instance.SharedLabelsWith(instance.Labels),
map[string]string{"service-type": "metrics"})
expectedServiceAnnotations := map[string]string{"prometheus.io/path":
"/metrics", "prometheus.io/port": "80", "prometheus.io/scheme": "http",
"prometheus.io/scrape": "true"}
service := expectService(t, g, requests, expectedMetricsRequest,
metricsSKey, expectedDeploymentLabels)
diff --git a/controllers/suite_test.go b/controllers/suite_test.go
index 4752253..77581c5 100644
--- a/controllers/suite_test.go
+++ b/controllers/suite_test.go
@@ -51,6 +51,14 @@ var additionalLables = map[string]string{
}
func TestMain(m *testing.M) {
+ // TODO: We can probably remove this once we upgrade our minimum
supported version of Kubernetes
+ customApiServerFlags := []string{
+ "--feature-gates=StartupProbe=true",
+ }
+
+ apiServerFlags := append([]string(nil),
envtest.DefaultKubeAPIServerFlags...)
+ apiServerFlags = append(apiServerFlags, customApiServerFlags...)
+
ctrl.SetLogger(zap.New(zap.UseDevMode(true)))
t := &envtest.Environment{
CRDDirectoryPaths: []string{
@@ -58,6 +66,7 @@ func TestMain(m *testing.M) {
filepath.Join("..", "config", "dependencies"),
},
AttachControlPlaneOutput: false, // set to true to get more
logging from the control plane
+ KubeAPIServerFlags: apiServerFlags,
}
solrv1beta1.AddToScheme(scheme.Scheme)
zkOp.AddToScheme(scheme.Scheme)
diff --git a/controllers/util/common.go b/controllers/util/common.go
index 7a3fd50..45074d4 100644
--- a/controllers/util/common.go
+++ b/controllers/util/common.go
@@ -132,6 +132,35 @@ func IsPVCOrphan(pvcName string, replicas int32) bool {
return int32(ordinal) >= replicas
}
+// customizeProbe builds the probe logic used for pod liveness, readiness,
startup checks
+func customizeProbe(initialProbe *corev1.Probe, customProbe corev1.Probe)
*corev1.Probe {
+ if customProbe.InitialDelaySeconds != 0 {
+ initialProbe.InitialDelaySeconds =
customProbe.InitialDelaySeconds
+ }
+
+ if customProbe.TimeoutSeconds != 0 {
+ initialProbe.TimeoutSeconds = customProbe.TimeoutSeconds
+ }
+
+ if customProbe.SuccessThreshold != 0 {
+ initialProbe.SuccessThreshold = customProbe.SuccessThreshold
+ }
+
+ if customProbe.FailureThreshold != 0 {
+ initialProbe.FailureThreshold = customProbe.FailureThreshold
+ }
+
+ if customProbe.PeriodSeconds != 0 {
+ initialProbe.PeriodSeconds = customProbe.PeriodSeconds
+ }
+
+ if customProbe.Handler.Exec != nil || customProbe.Handler.HTTPGet !=
nil || customProbe.Handler.TCPSocket != nil {
+ initialProbe.Handler = customProbe.Handler
+ }
+
+ return initialProbe
+}
+
// CopyConfigMapFields copies the owned fields from one ConfigMap to another
func CopyConfigMapFields(from, to *corev1.ConfigMap, logger logr.Logger) bool {
logger = logger.WithValues("kind", "configMap")
diff --git a/controllers/util/prometheus_exporter_util.go
b/controllers/util/prometheus_exporter_util.go
index f7aef36..03636fb 100644
--- a/controllers/util/prometheus_exporter_util.go
+++ b/controllers/util/prometheus_exporter_util.go
@@ -303,6 +303,7 @@ func
GenerateSolrPrometheusExporterDeployment(solrPrometheusExporter *solr.SolrP
deployment.Spec.Template.Spec.ImagePullSecrets = imagePullSecrets
if nil != customPodOptions {
+ metricsContainer := &deployment.Spec.Template.Spec.Containers[0]
if customPodOptions.ServiceAccountName != "" {
deployment.Spec.Template.Spec.ServiceAccountName =
customPodOptions.ServiceAccountName
}
@@ -311,12 +312,8 @@ func
GenerateSolrPrometheusExporterDeployment(solrPrometheusExporter *solr.SolrP
deployment.Spec.Template.Spec.Affinity =
customPodOptions.Affinity
}
- if customPodOptions.Resources.Requests != nil {
-
deployment.Spec.Template.Spec.Containers[0].Resources.Requests =
customPodOptions.Resources.Requests
- }
-
- if customPodOptions.Resources.Limits != nil {
-
deployment.Spec.Template.Spec.Containers[0].Resources.Limits =
customPodOptions.Resources.Limits
+ if customPodOptions.Resources.Limits != nil ||
customPodOptions.Resources.Requests != nil {
+ metricsContainer.Resources = customPodOptions.Resources
}
if customPodOptions.PodSecurityContext != nil {
@@ -338,6 +335,22 @@ func
GenerateSolrPrometheusExporterDeployment(solrPrometheusExporter *solr.SolrP
if customPodOptions.TerminationGracePeriodSeconds != nil {
deployment.Spec.Template.Spec.TerminationGracePeriodSeconds =
customPodOptions.TerminationGracePeriodSeconds
}
+
+ if customPodOptions.ReadinessProbe != nil {
+ // Default Prometheus Exporter container does not
contain a readinessProbe, so copy the livenessProbe
+ baseProbe := metricsContainer.LivenessProbe.DeepCopy()
+ metricsContainer.ReadinessProbe =
customizeProbe(baseProbe, *customPodOptions.ReadinessProbe)
+ }
+
+ if customPodOptions.StartupProbe != nil {
+ // Default Prometheus Exporter container does not
contain a startupProbe, so copy the livenessProbe
+ baseProbe := metricsContainer.LivenessProbe.DeepCopy()
+ metricsContainer.StartupProbe =
customizeProbe(baseProbe, *customPodOptions.StartupProbe)
+ }
+
+ if customPodOptions.LivenessProbe != nil {
+ metricsContainer.LivenessProbe =
customizeProbe(metricsContainer.LivenessProbe, *customPodOptions.LivenessProbe)
+ }
}
return deployment
diff --git a/controllers/util/solr_util.go b/controllers/util/solr_util.go
index 5c44f79..14276cb 100644
--- a/controllers/util/solr_util.go
+++ b/controllers/util/solr_util.go
@@ -63,24 +63,6 @@ const (
DefaultStatefulSetPodManagementPolicy = appsv1.ParallelPodManagement
- DefaultLivenessProbeInitialDelaySeconds = 20
- DefaultLivenessProbeTimeoutSeconds = 1
- DefaultLivenessProbeSuccessThreshold = 1
- DefaultLivenessProbeFailureThreshold = 3
- DefaultLivenessProbePeriodSeconds = 10
-
- DefaultReadinessProbeInitialDelaySeconds = 15
- DefaultReadinessProbeTimeoutSeconds = 1
- DefaultReadinessProbeSuccessThreshold = 1
- DefaultReadinessProbeFailureThreshold = 3
- DefaultReadinessProbePeriodSeconds = 5
-
- DefaultStartupProbeInitialDelaySeconds = 20
- DefaultStartupProbeTimeoutSeconds = 30
- DefaultStartupProbeSuccessThreshold = 1
- DefaultStartupProbeFailureThreshold = 15
- DefaultStartupProbePeriodSeconds = 10
-
DefaultKeyStorePath = "/var/solr/tls"
Pkcs12KeystoreFile = "keystore.p12"
DefaultWritableKeyStorePath = "/var/solr/tls/pkcs12"
@@ -445,19 +427,19 @@ func GenerateStatefulSet(solrCloud *solr.SolrCloud,
solrCloudStatus *solr.SolrCl
},
},
LivenessProbe: &corev1.Probe{
- InitialDelaySeconds:
DefaultLivenessProbeInitialDelaySeconds,
- TimeoutSeconds:
DefaultLivenessProbeTimeoutSeconds,
- SuccessThreshold:
DefaultLivenessProbeSuccessThreshold,
- FailureThreshold:
DefaultLivenessProbeFailureThreshold,
- PeriodSeconds:
DefaultLivenessProbePeriodSeconds,
+ InitialDelaySeconds: 20,
+ TimeoutSeconds: 1,
+ SuccessThreshold: 1,
+ FailureThreshold: 3,
+ PeriodSeconds: 10,
Handler: defaultHandler,
},
ReadinessProbe: &corev1.Probe{
- InitialDelaySeconds:
DefaultReadinessProbeInitialDelaySeconds,
- TimeoutSeconds:
DefaultReadinessProbeTimeoutSeconds,
- SuccessThreshold:
DefaultReadinessProbeSuccessThreshold,
- FailureThreshold:
DefaultReadinessProbeFailureThreshold,
- PeriodSeconds:
DefaultReadinessProbePeriodSeconds,
+ InitialDelaySeconds: 15,
+ TimeoutSeconds: 1,
+ SuccessThreshold: 1,
+ FailureThreshold: 3,
+ PeriodSeconds: 5,
Handler: defaultHandler,
},
VolumeMounts: volumeMounts,
@@ -552,6 +534,8 @@ func GenerateStatefulSet(solrCloud *solr.SolrCloud,
solrCloudStatus *solr.SolrCl
stateful.Spec.Template.Spec.ImagePullSecrets = imagePullSecrets
if nil != customPodOptions {
+ solrContainer := &stateful.Spec.Template.Spec.Containers[0]
+
if customPodOptions.ServiceAccountName != "" {
stateful.Spec.Template.Spec.ServiceAccountName =
customPodOptions.ServiceAccountName
}
@@ -561,7 +545,7 @@ func GenerateStatefulSet(solrCloud *solr.SolrCloud,
solrCloudStatus *solr.SolrCl
}
if customPodOptions.Resources.Limits != nil ||
customPodOptions.Resources.Requests != nil {
- stateful.Spec.Template.Spec.Containers[0].Resources =
customPodOptions.Resources
+ solrContainer.Resources = customPodOptions.Resources
}
if customPodOptions.PodSecurityContext != nil {
@@ -576,16 +560,21 @@ func GenerateStatefulSet(solrCloud *solr.SolrCloud,
solrCloudStatus *solr.SolrCl
stateful.Spec.Template.Spec.NodeSelector =
customPodOptions.NodeSelector
}
- if customPodOptions.LivenessProbe != nil {
- stateful.Spec.Template.Spec.Containers[0].LivenessProbe
= fillProbe(*customPodOptions.LivenessProbe,
DefaultLivenessProbeInitialDelaySeconds, DefaultLivenessProbeTimeoutSeconds,
DefaultLivenessProbeSuccessThreshold, DefaultLivenessProbeFailureThreshold,
DefaultLivenessProbePeriodSeconds, &defaultHandler)
+ if customPodOptions.StartupProbe != nil {
+ // Default Solr container does not contain a
startupProbe, so copy the livenessProbe
+ baseProbe := solrContainer.LivenessProbe.DeepCopy()
+ // Two options are different by default from the
livenessProbe
+ baseProbe.TimeoutSeconds = 30
+ baseProbe.FailureThreshold = 15
+ solrContainer.StartupProbe = customizeProbe(baseProbe,
*customPodOptions.StartupProbe)
}
- if customPodOptions.ReadinessProbe != nil {
-
stateful.Spec.Template.Spec.Containers[0].ReadinessProbe =
fillProbe(*customPodOptions.ReadinessProbe,
DefaultReadinessProbeInitialDelaySeconds, DefaultReadinessProbeTimeoutSeconds,
DefaultReadinessProbeSuccessThreshold, DefaultReadinessProbeFailureThreshold,
DefaultReadinessProbePeriodSeconds, &defaultHandler)
+ if customPodOptions.LivenessProbe != nil {
+ solrContainer.LivenessProbe =
customizeProbe(solrContainer.LivenessProbe, *customPodOptions.LivenessProbe)
}
- if customPodOptions.StartupProbe != nil {
- stateful.Spec.Template.Spec.Containers[0].StartupProbe
= fillProbe(*customPodOptions.StartupProbe,
DefaultStartupProbeInitialDelaySeconds, DefaultStartupProbeTimeoutSeconds,
DefaultStartupProbeSuccessThreshold, DefaultStartupProbeFailureThreshold,
DefaultStartupProbePeriodSeconds, &defaultHandler)
+ if customPodOptions.ReadinessProbe != nil {
+ solrContainer.ReadinessProbe =
customizeProbe(solrContainer.ReadinessProbe, *customPodOptions.ReadinessProbe)
}
if customPodOptions.PriorityClassName != "" {
@@ -686,44 +675,6 @@ func GenerateConfigMap(solrCloud *solr.SolrCloud)
*corev1.ConfigMap {
return configMap
}
-// fillProbe builds the probe logic used for pod liveness, readiness, startup
checks
-func fillProbe(customProbe corev1.Probe, defaultInitialDelaySeconds int32,
defaultTimeoutSeconds int32, defaultSuccessThreshold int32,
defaultFailureThreshold int32, defaultPeriodSeconds int32, defaultHandler
*corev1.Handler) *corev1.Probe {
- probe := &corev1.Probe{
- InitialDelaySeconds: defaultInitialDelaySeconds,
- TimeoutSeconds: defaultTimeoutSeconds,
- SuccessThreshold: defaultSuccessThreshold,
- FailureThreshold: defaultFailureThreshold,
- PeriodSeconds: defaultPeriodSeconds,
- Handler: *defaultHandler,
- }
-
- if customProbe.InitialDelaySeconds != 0 {
- probe.InitialDelaySeconds = customProbe.InitialDelaySeconds
- }
-
- if customProbe.TimeoutSeconds != 0 {
- probe.TimeoutSeconds = customProbe.TimeoutSeconds
- }
-
- if customProbe.SuccessThreshold != 0 {
- probe.SuccessThreshold = customProbe.SuccessThreshold
- }
-
- if customProbe.FailureThreshold != 0 {
- probe.FailureThreshold = customProbe.FailureThreshold
- }
-
- if customProbe.PeriodSeconds != 0 {
- probe.PeriodSeconds = customProbe.PeriodSeconds
- }
-
- if customProbe.Handler.Exec != nil || customProbe.Handler.HTTPGet !=
nil || customProbe.Handler.TCPSocket != nil {
- probe.Handler = customProbe.Handler
- }
-
- return probe
-}
-
// GenerateCommonService returns a new corev1.Service pointer generated for
the entire SolrCloud instance
// solrCloud: SolrCloud instance
func GenerateCommonService(solrCloud *solr.SolrCloud) *corev1.Service {
diff --git a/helm/solr-operator/Chart.yaml b/helm/solr-operator/Chart.yaml
index 9d61acd..6d791f6 100644
--- a/helm/solr-operator/Chart.yaml
+++ b/helm/solr-operator/Chart.yaml
@@ -98,6 +98,13 @@ annotations:
url: https://github.com/apache/solr-operator/issues/294
- name: Github PR
url: https://github.com/apache/solr-operator/pull/295
+ - kind: added
+ description: Ability to customize probes for PrometheusExporter
+ links:
+ - name: Github Issue
+ url: https://github.com/apache/solr-operator/issues/282
+ - name: Github PR
+ url: https://github.com/apache/solr-operator/pull/297
artifacthub.io/images: |
- name: solr-operator
image: apache/solr-operator:v0.4.0-prerelease
