Skip to content

Commit 497630e

Browse files
HoustonPutmanHoustonPutman
authored
Add terminationGracePeriod option, and use it when killing Solr pods. (#226)
terminationGracePeriod used to be statically set to 10 seconds. Now it defaults to 30 seconds, but is configurable. SOLR_STOP_WAIT will be set to the terminationGracePeriod, minus a few seconds, so that kubernetes and Solr are on the same page.
1 parent 5c08f80 commit 497630e

12 files changed

+138
-55
lines changed

‎api/v1beta1/common_types.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,11 @@ type PodOptions struct {
124124
// solr image.
125125
// +optional
126126
ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"`
127+
128+
// Optional duration in seconds the pod needs to terminate gracefully.
129+
// +kubebuilder:validation:Minimum=10
130+
// +optional
131+
TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty"`
127132
}
128133

129134
// ServiceOptions defines custom options for services

‎api/v1beta1/zz_generated.deepcopy.go

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎config/crd/bases/solr.apache.org_solrclouds.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2417,6 +2417,11 @@ spec:
24172417
format: int32
24182418
type: integer
24192419
type: object
2420+
terminationGracePeriodSeconds:
2421+
description: Optional duration in seconds the pod needs to terminate gracefully.
2422+
format: int64
2423+
minimum: 10
2424+
type: integer
24202425
tolerations:
24212426
description: Tolerations to be added for the StatefulSet.
24222427
items:

‎config/crd/bases/solr.apache.org_solrprometheusexporters.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2346,6 +2346,11 @@ spec:
23462346
format: int32
23472347
type: integer
23482348
type: object
2349+
terminationGracePeriodSeconds:
2350+
description: Optional duration in seconds the pod needs to terminate gracefully.
2351+
format: int64
2352+
minimum: 10
2353+
type: integer
23492354
tolerations:
23502355
description: Tolerations to be added for the StatefulSet.
23512356
items:

‎controllers/controller_utils_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -543,7 +543,8 @@ var (
543543
{Name: "ADDITIONAL_SECRET_1"},
544544
{Name: "ADDITIONAL_SECRET_2"},
545545
}
546-
extraVars = []corev1.EnvVar{
546+
testTerminationGracePeriodSeconds = int64(50)
547+
extraVars = []corev1.EnvVar{
547548
{
548549
Name: "VAR_1",
549550
Value: "VAL_1",

‎controllers/solrcloud_controller_test.go

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ package controllers
2020
import (
2121
"crypto/md5"
2222
"fmt"
23+
"strconv"
2324
"time"
2425

2526
solr "github.com/apache/lucene-solr-operator/api/v1beta1"
@@ -189,15 +190,16 @@ func TestCustomKubeOptionsCloudReconcile(t *testing.T) {
189190
SolrGCTune: "gc Options",
190191
CustomSolrKubeOptions: solr.CustomSolrKubeOptions{
191192
PodOptions: &solr.PodOptions{
192-
Annotations: testPodAnnotations,
193-
Labels: testPodLabels,
194-
Tolerations: testTolerations,
195-
NodeSelector: testNodeSelectors,
196-
LivenessProbe: testProbeLivenessNonDefaults,
197-
ReadinessProbe: testProbeReadinessNonDefaults,
198-
StartupProbe: testProbeStartup,
199-
PriorityClassName: testPriorityClass,
200-
ImagePullSecrets: testAdditionalImagePullSecrets,
193+
Annotations: testPodAnnotations,
194+
Labels: testPodLabels,
195+
Tolerations: testTolerations,
196+
NodeSelector: testNodeSelectors,
197+
LivenessProbe: testProbeLivenessNonDefaults,
198+
ReadinessProbe: testProbeReadinessNonDefaults,
199+
StartupProbe: testProbeStartup,
200+
PriorityClassName: testPriorityClass,
201+
ImagePullSecrets: testAdditionalImagePullSecrets,
202+
TerminationGracePeriodSeconds: &testTerminationGracePeriodSeconds,
201203
},
202204
StatefulSetOptions: &solr.StatefulSetOptions{
203205
Annotations: testSSAnnotations,
@@ -268,6 +270,7 @@ func TestCustomKubeOptionsCloudReconcile(t *testing.T) {
268270
"SOLR_NODE_PORT": "8983",
269271
"GC_TUNE": "gc Options",
270272
"SOLR_OPTS": "-DhostPort=$(SOLR_NODE_PORT)",
273+
"SOLR_STOP_WAIT": strconv.FormatInt(testTerminationGracePeriodSeconds-5, 10),
271274
}
272275
expectedStatefulSetLabels := util.MergeLabelsOrAnnotations(instance.SharedLabelsWith(instance.Labels), map[string]string{"technology": "solr-cloud"})
273276
expectedStatefulSetAnnotations := map[string]string{util.SolrZKConnectionStringAnnotation: "host:7271/"}
@@ -283,6 +286,7 @@ func TestCustomKubeOptionsCloudReconcile(t *testing.T) {
283286
testPodTolerations(t, testTolerations, statefulSet.Spec.Template.Spec.Tolerations)
284287
assert.EqualValues(t, testPriorityClass, statefulSet.Spec.Template.Spec.PriorityClassName, "Incorrect Priority class name for Pod Spec")
285288
assert.ElementsMatch(t, append(testAdditionalImagePullSecrets, corev1.LocalObjectReference{Name: testImagePullSecretName}), statefulSet.Spec.Template.Spec.ImagePullSecrets, "Incorrect imagePullSecrets")
289+
assert.EqualValues(t, &testTerminationGracePeriodSeconds, statefulSet.Spec.Template.Spec.TerminationGracePeriodSeconds, "Incorrect terminationGracePeriodSeconds")
286290

287291
// Check the update strategy
288292
assert.EqualValues(t, appsv1.RollingUpdateStatefulSetStrategyType, statefulSet.Spec.UpdateStrategy.Type, "Incorrect statefulset update strategy")

‎controllers/solrprometheusexporter_controller_test.go

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -56,14 +56,15 @@ func TestMetricsReconcileWithoutExporterConfig(t *testing.T) {
5656
Spec: solr.SolrPrometheusExporterSpec{
5757
CustomKubeOptions: solr.CustomExporterKubeOptions{
5858
PodOptions: &solr.PodOptions{
59-
EnvVariables: extraVars,
60-
PodSecurityContext: &podSecurityContext,
61-
Volumes: extraVolumes,
62-
Affinity: affinity,
63-
Resources: resources,
64-
SidecarContainers: extraContainers2,
65-
InitContainers: extraContainers1,
66-
ImagePullSecrets: testAdditionalImagePullSecrets,
59+
EnvVariables: extraVars,
60+
PodSecurityContext: &podSecurityContext,
61+
Volumes: extraVolumes,
62+
Affinity: affinity,
63+
Resources: resources,
64+
SidecarContainers: extraContainers2,
65+
InitContainers: extraContainers1,
66+
ImagePullSecrets: testAdditionalImagePullSecrets,
67+
TerminationGracePeriodSeconds: &testTerminationGracePeriodSeconds,
6768
},
6869
},
6970
ExporterEntrypoint: "/test/entry-point",
@@ -129,6 +130,7 @@ func TestMetricsReconcileWithoutExporterConfig(t *testing.T) {
129130
assert.Equal(t, extraVolumes[0].Name, deployment.Spec.Template.Spec.Volumes[0].Name, "Additional Volume from podOptions not loaded into pod properly.")
130131
assert.Equal(t, extraVolumes[0].Source, deployment.Spec.Template.Spec.Volumes[0].VolumeSource, "Additional Volume from podOptions not loaded into pod properly.")
131132
assert.ElementsMatch(t, append(testAdditionalImagePullSecrets, corev1.LocalObjectReference{Name: testImagePullSecretName}), deployment.Spec.Template.Spec.ImagePullSecrets, "Incorrect imagePullSecrets")
133+
assert.EqualValues(t, &testTerminationGracePeriodSeconds, deployment.Spec.Template.Spec.TerminationGracePeriodSeconds, "Incorrect terminationGracePeriodSeconds")
132134

133135
service := expectService(t, g, requests, expectedMetricsRequest, metricsSKey, deployment.Spec.Template.Labels)
134136
assert.Equal(t, "true", service.Annotations["prometheus.io/scrape"], "Metrics Service Prometheus scraping is not enabled.")

‎controllers/util/common.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -422,6 +422,12 @@ func CopyPodTemplates(from, to *corev1.PodTemplateSpec, basePath string, logger
422422
to.Spec.PriorityClassName = from.Spec.PriorityClassName
423423
}
424424

425+
if !DeepEqualWithNils(to.Spec.TerminationGracePeriodSeconds, from.Spec.TerminationGracePeriodSeconds) {
426+
requireUpdate = true
427+
logger.Info("Update required because field changed", "field", basePath+"Spec.TerminationGracePeriodSeconds", "from", to.Spec.TerminationGracePeriodSeconds, "to", from.Spec.TerminationGracePeriodSeconds)
428+
to.Spec.TerminationGracePeriodSeconds = from.Spec.TerminationGracePeriodSeconds
429+
}
430+
425431
return requireUpdate
426432
}
427433

‎controllers/util/prometheus_exporter_util.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -330,6 +330,10 @@ func GenerateSolrPrometheusExporterDeployment(solrPrometheusExporter *solr.SolrP
330330
if customPodOptions.PriorityClassName != "" {
331331
deployment.Spec.Template.Spec.PriorityClassName = customPodOptions.PriorityClassName
332332
}
333+
334+
if customPodOptions.TerminationGracePeriodSeconds != nil {
335+
deployment.Spec.Template.Spec.TerminationGracePeriodSeconds = customPodOptions.TerminationGracePeriodSeconds
336+
}
333337
}
334338

335339
return deployment

‎controllers/util/solr_util.go

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ const (
9292
// storage: the size of the storage for the SolrCloud instance (e.g. 100Gi)
9393
// zkConnectionString: the connectionString of the ZK instance to connect to
9494
func GenerateStatefulSet(solrCloud *solr.SolrCloud, solrCloudStatus *solr.SolrCloudStatus, hostNameIPs map[string]string, reconcileConfigInfo map[string]string, createPkcs12InitContainer bool, tlsCertMd5 string) *appsv1.StatefulSet {
95-
gracePeriodTerm := int64(10)
95+
terminationGracePeriod := int64(60)
9696
solrPodPort := solrCloud.Spec.SolrAddressability.PodPort
9797
fsGroup := int64(DefaultSolrGroup)
9898
defaultMode := int32(420)
@@ -133,6 +133,10 @@ func GenerateStatefulSet(solrCloud *solr.SolrCloud, solrCloudStatus *solr.SolrCl
133133
if nil != customPodOptions {
134134
podLabels = MergeLabelsOrAnnotations(podLabels, customPodOptions.Labels)
135135
podAnnotations = customPodOptions.Annotations
136+
137+
if customPodOptions.TerminationGracePeriodSeconds != nil {
138+
terminationGracePeriod = *customPodOptions.TerminationGracePeriodSeconds
139+
}
136140
}
137141

138142
// Keep track of the SolrOpts that the Solr Operator needs to set
@@ -274,6 +278,12 @@ func GenerateStatefulSet(solrCloud *solr.SolrCloud, solrCloudStatus *solr.SolrCl
274278
solrHostName := solrCloud.AdvertisedNodeHost("$(POD_HOSTNAME)")
275279
solrAdressingPort := solrCloud.NodePort()
276280

281+
// Solr can take longer than SOLR_STOP_WAIT to run solr stop, give it a few extra seconds before forcefully killing the pod.
282+
solrStopWait := terminationGracePeriod - 5
283+
if solrStopWait < 0 {
284+
solrStopWait = 0
285+
}
286+
277287
// Environment Variables
278288
envVars := []corev1.EnvVar{
279289
{
@@ -315,6 +325,10 @@ func GenerateStatefulSet(solrCloud *solr.SolrCloud, solrCloudStatus *solr.SolrCl
315325
Name: "GC_TUNE",
316326
Value: solrCloud.Spec.SolrGCTune,
317327
},
328+
{
329+
Name: "SOLR_STOP_WAIT",
330+
Value: strconv.FormatInt(solrStopWait, 10),
331+
},
318332
}
319333

320334
// Add all necessary information for connection to Zookeeper
@@ -530,7 +544,7 @@ func GenerateStatefulSet(solrCloud *solr.SolrCloud, solrCloudStatus *solr.SolrCl
530544
},
531545

532546
Spec: corev1.PodSpec{
533-
TerminationGracePeriodSeconds: &gracePeriodTerm,
547+
TerminationGracePeriodSeconds: &terminationGracePeriod,
534548
SecurityContext: &corev1.PodSecurityContext{
535549
FSGroup: &fsGroup,
536550
},

0 commit comments

Comments
 (0)