Skip to content

Commit 032d70a

Browse files
bsankarabsankara
authored
Merge pull request #127 from silvermullet/feat/configurable_checks
Adding configurable liveness and readiness checks
2 parents 4d05a34 + 4b093e7 commit 032d70a

9 files changed

+1533
-31
lines changed

‎api/v1beta1/common_types.go

+12
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,18 @@ type PodOptions struct {
8383
// Node Selector to be added for the StatefulSet.
8484
// +optional
8585
NodeSelector map[string]string `json:"nodeSelector,omitempty"`
86+
87+
// Liveness probe parameters
88+
// +optional
89+
LivenessProbe *corev1.Probe `json:"livenessProbe,omitempty"`
90+
91+
// Readiness probe parameters
92+
// +optional
93+
ReadinessProbe *corev1.Probe `json:"readinessProbe,omitempty"`
94+
95+
// Startup probe parameters
96+
// +optional
97+
StartupProbe *corev1.Probe `json:"startupProbe,omitempty"`
8698
}
8799

88100
// ServiceOptions defines custom options for services

‎api/v1beta1/zz_generated.deepcopy.go

+15
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎config/crd/bases/solr.bloomberg.com_solrclouds.yaml

+339
Large diffs are not rendered by default.

‎config/crd/bases/solr.bloomberg.com_solrprometheusexporters.yaml

+339
Large diffs are not rendered by default.

‎controllers/controller_utils_test.go

+47
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import (
3131
"k8s.io/apimachinery/pkg/api/resource"
3232
"k8s.io/apimachinery/pkg/runtime"
3333
"k8s.io/apimachinery/pkg/types"
34+
"k8s.io/apimachinery/pkg/util/intstr"
3435
"sigs.k8s.io/controller-runtime/pkg/client"
3536
"sigs.k8s.io/controller-runtime/pkg/reconcile"
3637
)
@@ -191,6 +192,10 @@ func testPodTolerations(t *testing.T, expectedTolerations []corev1.Toleration, f
191192
assert.True(t, reflect.DeepEqual(expectedTolerations, foundTolerations), "Expected tolerations and found tolerations don't match")
192193
}
193194

195+
func testPodProbe(t *testing.T, expectedProbe *corev1.Probe, foundProbe *corev1.Probe) {
196+
assert.True(t, reflect.DeepEqual(expectedProbe, foundProbe), "Expected probe and found probe don't match")
197+
}
198+
194199
func testMapsEqual(t *testing.T, mapName string, expected map[string]string, found map[string]string) {
195200
assert.Equal(t, expected, found, "Expected and found %s are not the same", mapName)
196201
}
@@ -324,6 +329,48 @@ var (
324329
"beta.kubernetes.io/os": "linux",
325330
"solrclouds": "true",
326331
}
332+
testProbeLivenessNonDefaults = &corev1.Probe{
333+
InitialDelaySeconds: 20,
334+
TimeoutSeconds: 1,
335+
SuccessThreshold: 1,
336+
FailureThreshold: 3,
337+
PeriodSeconds: 10,
338+
Handler: corev1.Handler{
339+
HTTPGet: &corev1.HTTPGetAction{
340+
Scheme: corev1.URISchemeHTTP,
341+
Path: "/solr/admin/info/system",
342+
Port: intstr.FromInt(8983),
343+
},
344+
},
345+
}
346+
testProbeReadinessNonDefaults = &corev1.Probe{
347+
InitialDelaySeconds: 15,
348+
TimeoutSeconds: 1,
349+
SuccessThreshold: 1,
350+
FailureThreshold: 3,
351+
PeriodSeconds: 5,
352+
Handler: corev1.Handler{
353+
HTTPGet: &corev1.HTTPGetAction{
354+
Scheme: corev1.URISchemeHTTP,
355+
Path: "/solr/admin/info/system",
356+
Port: intstr.FromInt(8983),
357+
},
358+
},
359+
}
360+
testProbeStartup = &corev1.Probe{
361+
InitialDelaySeconds: 1,
362+
TimeoutSeconds: 1,
363+
SuccessThreshold: 1,
364+
FailureThreshold: 5,
365+
PeriodSeconds: 5,
366+
Handler: corev1.Handler{
367+
Exec: &corev1.ExecAction{
368+
Command: []string{
369+
"ls",
370+
},
371+
},
372+
},
373+
}
327374
testTolerations = []corev1.Toleration{
328375
{
329376
Effect: "NoSchedule",

‎controllers/solrcloud_controller_test.go

+9-4
Original file line numberDiff line numberDiff line change
@@ -168,10 +168,13 @@ func TestCustomKubeOptionsCloudReconcile(t *testing.T) {
168168
SolrGCTune: "gc Options",
169169
CustomSolrKubeOptions: solr.CustomSolrKubeOptions{
170170
PodOptions: &solr.PodOptions{
171-
Annotations: testPodAnnotations,
172-
Labels: testPodLabels,
173-
Tolerations: testTolerations,
174-
NodeSelector: testNodeSelectors,
171+
Annotations: testPodAnnotations,
172+
Labels: testPodLabels,
173+
Tolerations: testTolerations,
174+
NodeSelector: testNodeSelectors,
175+
LivenessProbe: testProbeLivenessNonDefaults,
176+
ReadinessProbe: testProbeReadinessNonDefaults,
177+
StartupProbe: testProbeStartup,
175178
},
176179
StatefulSetOptions: &solr.StatefulSetOptions{
177180
Annotations: testSSAnnotations,
@@ -251,6 +254,8 @@ func TestCustomKubeOptionsCloudReconcile(t *testing.T) {
251254
testMapsEqual(t, "pod labels", util.MergeLabelsOrAnnotations(expectedStatefulSetLabels, testPodLabels), statefulSet.Spec.Template.ObjectMeta.Labels)
252255
testMapsEqual(t, "pod annotations", testPodAnnotations, statefulSet.Spec.Template.Annotations)
253256
testMapsEqual(t, "pod node selectors", testNodeSelectors, statefulSet.Spec.Template.Spec.NodeSelector)
257+
testPodProbe(t, testProbeLivenessNonDefaults, statefulSet.Spec.Template.Spec.Containers[0].LivenessProbe)
258+
testPodProbe(t, testProbeReadinessNonDefaults, statefulSet.Spec.Template.Spec.Containers[0].ReadinessProbe)
254259
testPodTolerations(t, testTolerations, statefulSet.Spec.Template.Spec.Tolerations)
255260

256261
// Check the client Service

‎controllers/util/solr_util.go

+91-27
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,24 @@ const (
4141
BackupRestoreVolume = "backup-restore"
4242

4343
SolrZKConnectionStringAnnotation = "solr.apache.org/zkConnectionString"
44+
45+
DefaultLivenessProbeInitialDelaySeconds = 20
46+
DefaultLivenessProbeTimeoutSeconds = 1
47+
DefaultLivenessProbeSuccessThreshold = 1
48+
DefaultLivenessProbeFailureThreshold = 3
49+
DefaultLivenessProbePeriodSeconds = 10
50+
51+
DefaultReadinessProbeInitialDelaySeconds = 15
52+
DefaultReadinessProbeTimeoutSeconds = 1
53+
DefaultReadinessProbeSuccessThreshold = 1
54+
DefaultReadinessProbeFailureThreshold = 3
55+
DefaultReadinessProbePeriodSeconds = 5
56+
57+
DefaultStartupProbeInitialDelaySeconds = 20
58+
DefaultStartupProbeTimeoutSeconds = 30
59+
DefaultStartupProbeSuccessThreshold = 1
60+
DefaultStartupProbeFailureThreshold = 15
61+
DefaultStartupProbePeriodSeconds = 10
4462
)
4563

4664
// GenerateStatefulSet returns a new appsv1.StatefulSet pointer generated for the SolrCloud instance
@@ -53,6 +71,13 @@ func GenerateStatefulSet(solrCloud *solr.SolrCloud, solrCloudStatus *solr.SolrCl
5371
solrPodPort := solrCloud.Spec.SolrAddressability.PodPort
5472
fsGroup := int64(solrPodPort)
5573
defaultMode := int32(420)
74+
defaultHandler := corev1.Handler{
75+
HTTPGet: &corev1.HTTPGetAction{
76+
Scheme: corev1.URISchemeHTTP,
77+
Path: "/solr/admin/info/system",
78+
Port: intstr.FromInt(solrPodPort),
79+
},
80+
}
5681

5782
labels := solrCloud.SharedLabelsWith(solrCloud.GetLabels())
5883
selectorLabels := solrCloud.SharedLabels()
@@ -276,37 +301,25 @@ func GenerateStatefulSet(solrCloud *solr.SolrCloud, solrCloudStatus *solr.SolrCl
276301
Protocol: "TCP",
277302
},
278303
},
279-
VolumeMounts: volumeMounts,
280-
Args: []string{"-DhostPort=" + strconv.Itoa(solrAdressingPort)},
281-
Env: envVars,
282304
LivenessProbe: &corev1.Probe{
283-
InitialDelaySeconds: 20,
284-
TimeoutSeconds: 1,
285-
SuccessThreshold: 1,
286-
FailureThreshold: 3,
287-
PeriodSeconds: 10,
288-
Handler: corev1.Handler{
289-
HTTPGet: &corev1.HTTPGetAction{
290-
Scheme: corev1.URISchemeHTTP,
291-
Path: "/solr/admin/info/system",
292-
Port: intstr.FromInt(solrPodPort),
293-
},
294-
},
305+
InitialDelaySeconds: DefaultLivenessProbeInitialDelaySeconds,
306+
TimeoutSeconds: DefaultLivenessProbeTimeoutSeconds,
307+
SuccessThreshold: DefaultLivenessProbeSuccessThreshold,
308+
FailureThreshold: DefaultLivenessProbeFailureThreshold,
309+
PeriodSeconds: DefaultLivenessProbePeriodSeconds,
310+
Handler: defaultHandler,
295311
},
296312
ReadinessProbe: &corev1.Probe{
297-
InitialDelaySeconds: 15,
298-
TimeoutSeconds: 1,
299-
SuccessThreshold: 1,
300-
FailureThreshold: 3,
301-
PeriodSeconds: 5,
302-
Handler: corev1.Handler{
303-
HTTPGet: &corev1.HTTPGetAction{
304-
Scheme: corev1.URISchemeHTTP,
305-
Path: "/solr/admin/info/system",
306-
Port: intstr.FromInt(solrPodPort),
307-
},
308-
},
313+
InitialDelaySeconds: DefaultReadinessProbeInitialDelaySeconds,
314+
TimeoutSeconds: DefaultReadinessProbeTimeoutSeconds,
315+
SuccessThreshold: DefaultReadinessProbeSuccessThreshold,
316+
FailureThreshold: DefaultReadinessProbeFailureThreshold,
317+
PeriodSeconds: DefaultReadinessProbePeriodSeconds,
318+
Handler: defaultHandler,
309319
},
320+
VolumeMounts: volumeMounts,
321+
Args: []string{"-DhostPort=" + strconv.Itoa(solrAdressingPort)},
322+
Env: envVars,
310323
TerminationMessagePath: "/dev/termination-log",
311324
TerminationMessagePolicy: "File",
312325
},
@@ -353,6 +366,19 @@ func GenerateStatefulSet(solrCloud *solr.SolrCloud, solrCloudStatus *solr.SolrCl
353366
if customPodOptions.NodeSelector != nil {
354367
stateful.Spec.Template.Spec.NodeSelector = customPodOptions.NodeSelector
355368
}
369+
370+
if customPodOptions.LivenessProbe != nil {
371+
stateful.Spec.Template.Spec.Containers[0].LivenessProbe = fillProbe(*customPodOptions.LivenessProbe, DefaultLivenessProbeInitialDelaySeconds, DefaultLivenessProbeTimeoutSeconds, DefaultLivenessProbeSuccessThreshold, DefaultLivenessProbeFailureThreshold, DefaultLivenessProbePeriodSeconds, &defaultHandler)
372+
}
373+
374+
if customPodOptions.ReadinessProbe != nil {
375+
stateful.Spec.Template.Spec.Containers[0].ReadinessProbe = fillProbe(*customPodOptions.ReadinessProbe, DefaultReadinessProbeInitialDelaySeconds, DefaultReadinessProbeTimeoutSeconds, DefaultReadinessProbeSuccessThreshold, DefaultReadinessProbeFailureThreshold, DefaultReadinessProbePeriodSeconds, &defaultHandler)
376+
}
377+
378+
if customPodOptions.StartupProbe != nil {
379+
stateful.Spec.Template.Spec.Containers[0].StartupProbe = fillProbe(*customPodOptions.StartupProbe, DefaultStartupProbeInitialDelaySeconds, DefaultStartupProbeTimeoutSeconds, DefaultStartupProbeSuccessThreshold, DefaultStartupProbeFailureThreshold, DefaultStartupProbePeriodSeconds, &defaultHandler)
380+
}
381+
356382
}
357383

358384
return stateful
@@ -526,6 +552,44 @@ func CopyConfigMapFields(from, to *corev1.ConfigMap) bool {
526552
return requireUpdate
527553
}
528554

555+
// fillProbe builds the probe logic used for pod liveness, readiness, startup checks
556+
func fillProbe(customSolrKubeOptions corev1.Probe, defaultInitialDelaySeconds int32, defaultTimeoutSeconds int32, defaultSuccessThreshold int32, defaultFailureThreshold int32, defaultPeriodSeconds int32, defaultHandler *corev1.Handler) *corev1.Probe {
557+
probe := &corev1.Probe{
558+
InitialDelaySeconds: defaultInitialDelaySeconds,
559+
TimeoutSeconds: defaultTimeoutSeconds,
560+
SuccessThreshold: defaultSuccessThreshold,
561+
FailureThreshold: defaultFailureThreshold,
562+
PeriodSeconds: defaultPeriodSeconds,
563+
Handler: *defaultHandler,
564+
}
565+
566+
if customSolrKubeOptions.InitialDelaySeconds != 0 {
567+
probe.InitialDelaySeconds = customSolrKubeOptions.InitialDelaySeconds
568+
}
569+
570+
if customSolrKubeOptions.TimeoutSeconds != 0 {
571+
probe.TimeoutSeconds = customSolrKubeOptions.TimeoutSeconds
572+
}
573+
574+
if customSolrKubeOptions.SuccessThreshold != 0 {
575+
probe.SuccessThreshold = customSolrKubeOptions.SuccessThreshold
576+
}
577+
578+
if customSolrKubeOptions.FailureThreshold != 0 {
579+
probe.FailureThreshold = customSolrKubeOptions.FailureThreshold
580+
}
581+
582+
if customSolrKubeOptions.PeriodSeconds != 0 {
583+
probe.PeriodSeconds = customSolrKubeOptions.PeriodSeconds
584+
}
585+
586+
if customSolrKubeOptions.Handler.Exec != nil || customSolrKubeOptions.Handler.HTTPGet != nil {
587+
probe.Handler = customSolrKubeOptions.Handler
588+
}
589+
590+
return probe
591+
}
592+
529593
// GenerateCommonService returns a new corev1.Service pointer generated for the entire SolrCloud instance
530594
// solrCloud: SolrCloud instance
531595
func GenerateCommonService(solrCloud *solr.SolrCloud) *corev1.Service {

‎go.sum

+3
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,10 @@ github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdko
4949
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
5050
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
5151
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
52+
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
5253
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
5354
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
55+
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E=
5456
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
5557
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
5658
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
@@ -647,6 +649,7 @@ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8
647649
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
648650
google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
649651
google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
652+
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
650653
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
651654
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
652655
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=

0 commit comments

Comments
 (0)