Adding jitter based on percent to default token expiration to avoid t…

…hundering herd

pkg/constants.go

@@ -15,9 +15,10 @@ permissions and limitations under the License.
 package pkg
 
 const (
-	// Default token expiration in seconds if none is defined,
-	// which is 24hrs as that is max for EKS
-	DefaultTokenExpiration = int64(86400)
+	// 24hrs as that is max for EKS
+	MaxTokenExpiration = int64(86400)
+	// Default token expiration in seconds if none is defined, 22hrs
+	DefaultTokenExpiration = int64(79200)
 	// 10mins is min for kube-apiserver
 	MinTokenExpiration = int64(600)
 

pkg/handler/handler.go

@@ -20,6 +20,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"math/rand"
 	"net/http"
 	"path/filepath"
 	"strconv"
@@ -417,6 +418,14 @@ func (m *Modifier) buildPodPatchConfig(pod *corev1.Pod) *podPatchConfig {
 		regionalSTS, tokenExpiration := m.Cache.GetCommonConfigurations(pod.Spec.ServiceAccountName, pod.Namespace)
 		tokenExpiration, containersToSkip := m.parsePodAnnotations(pod, tokenExpiration)
 
+		if tokenExpiration == pkg.DefaultTokenExpiration {
+			klog.V(4).Infof("Adding jitter to default token expiration")
+			var err error
+			tokenExpiration, err = addJitter(tokenExpiration, 5, pkg.MinTokenExpiration, pkg.MaxTokenExpiration)
+			if err != nil {
+				klog.Errorf("Error adding jitter to default token expiration: %v", err)
+			}
+		}
 		webhookPodCount.WithLabelValues("container_credentials").Inc()
 
 		return &podPatchConfig{
@@ -479,6 +488,26 @@ func (m *Modifier) buildPodPatchConfig(pod *corev1.Pod) *podPatchConfig {
 	return nil
 }
 
+func addJitter(val int64, jitterPercent int64, min int64, max int64) (int64, error) {
+	if max < min {
+		return val, error(fmt.Errorf("max value %d is less than min value %d, cannot add jitter", max, min))
+	}
+
+	jitterFactor := float64(jitterPercent) / 100.0
+	jitterMin := int64(float64(val) - (float64(val) * jitterFactor))
+	if jitterMin < min {
+		jitterMin = min
+	}
+	jitterMax := int64(float64(val) + (float64(val) * jitterFactor))
+	if jitterMax > max {
+		jitterMax = max
+	}
+
+	valWithJitter := rand.Int63n(jitterMax - jitterMin + 1) + jitterMin
+
+	return valWithJitter, nil
+}
+
 // MutatePod takes a AdmissionReview, mutates the pod, and returns an AdmissionResponse
 func (m *Modifier) MutatePod(ar *v1beta1.AdmissionReview) *v1beta1.AdmissionResponse {
 	badRequest := &v1beta1.AdmissionResponse{

pkg/handler/handler_test.go

@@ -180,6 +180,24 @@ func serializeAdmissionReview(t *testing.T, want *v1beta1.AdmissionReview) []byt
 	return wantedBytes
 }
 
+func TestAddJitterMinMax(t *testing.T) {
+	var (
+		min int64
+		max int64
+	)
+	min, max = 8, 11
+	for i := 0; i < 10; i++ {
+		jitter, err := addJitter(10, 1000, min, max)
+		assert.True(t, jitter >= min && jitter <= max)
+		assert.True(t, err == nil)
+	}
+}
+
+func TestAddJitterMinGTMax(t *testing.T) {
+	_, err := addJitter(10, 1000, 11, 8)
+	assert.True(t, err != nil)
+}
+
 func TestModifierHandler(t *testing.T) {
 	testServiceAccount := &corev1.ServiceAccount{}
 	testServiceAccount.Name = "default"