Skip to content

Commit 955b30b

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #12227 from k8s-infra-cherrypick-robot/cherry-pick-11520-to-release-1.9
[release-1.9] 🐛 Recreate bootstrap token if it was cleaned up
2 parents 324899f + 0a5229e commit 955b30b

File tree

2 files changed

+177
-12
lines changed

2 files changed

+177
-12
lines changed

‎bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go‎

Lines changed: 28 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -346,7 +346,7 @@ func (r *KubeadmConfigReconciler) reconcile(ctx context.Context, scope *Scope, c
346346
// If the BootstrapToken has been generated for a join but the config owner has no nodeRefs,
347347
// this indicates that the node has not yet joined and the token in the join config has not
348348
// been consumed and it may need a refresh.
349-
return r.refreshBootstrapTokenIfNeeded(ctx, config, cluster)
349+
return r.refreshBootstrapTokenIfNeeded(ctx, config, cluster, scope)
350350
}
351351
if configOwner.IsMachinePool() {
352352
// If the BootstrapToken has been generated and infrastructure is ready but the configOwner is a MachinePool,
@@ -384,7 +384,7 @@ func (r *KubeadmConfigReconciler) reconcile(ctx context.Context, scope *Scope, c
384384
return r.joinWorker(ctx, scope)
385385
}
386386

387-
func (r *KubeadmConfigReconciler) refreshBootstrapTokenIfNeeded(ctx context.Context, config *bootstrapv1.KubeadmConfig, cluster *clusterv1.Cluster) (ctrl.Result, error) {
387+
func (r *KubeadmConfigReconciler) refreshBootstrapTokenIfNeeded(ctx context.Context, config *bootstrapv1.KubeadmConfig, cluster *clusterv1.Cluster, scope *Scope) (ctrl.Result, error) {
388388
log := ctrl.LoggerFrom(ctx)
389389
token := config.Spec.JoinConfiguration.Discovery.BootstrapToken.Token
390390

@@ -395,6 +395,11 @@ func (r *KubeadmConfigReconciler) refreshBootstrapTokenIfNeeded(ctx context.Cont
395395

396396
secret, err := getToken(ctx, remoteClient, token)
397397
if err != nil {
398+
if apierrors.IsNotFound(err) && scope.ConfigOwner.IsMachinePool() {
399+
log.Info("Bootstrap token secret not found, triggering creation of new token")
400+
config.Spec.JoinConfiguration.Discovery.BootstrapToken.Token = ""
401+
return r.recreateBootstrapToken(ctx, config, scope, remoteClient)
402+
}
398403
return ctrl.Result{}, errors.Wrapf(err, "failed to get bootstrap token secret in order to refresh it")
399404
}
400405
log = log.WithValues("Secret", klog.KObj(secret))
@@ -425,13 +430,33 @@ func (r *KubeadmConfigReconciler) refreshBootstrapTokenIfNeeded(ctx context.Cont
425430
log.Info("Refreshing token until the infrastructure has a chance to consume it", "oldExpiration", secretExpiration, "newExpiration", newExpiration)
426431
err = remoteClient.Update(ctx, secret)
427432
if err != nil {
433+
if apierrors.IsNotFound(err) && scope.ConfigOwner.IsMachinePool() {
434+
log.Info("Bootstrap token secret not found, triggering creation of new token")
435+
config.Spec.JoinConfiguration.Discovery.BootstrapToken.Token = ""
436+
return r.recreateBootstrapToken(ctx, config, scope, remoteClient)
437+
}
428438
return ctrl.Result{}, errors.Wrapf(err, "failed to refresh bootstrap token")
429439
}
430440
return ctrl.Result{
431441
RequeueAfter: r.tokenCheckRefreshOrRotationInterval(),
432442
}, nil
433443
}
434444

445+
func (r *KubeadmConfigReconciler) recreateBootstrapToken(ctx context.Context, config *bootstrapv1.KubeadmConfig, scope *Scope, remoteClient client.Client) (ctrl.Result, error) {
446+
log := ctrl.LoggerFrom(ctx)
447+
448+
token, err := createToken(ctx, remoteClient, r.TokenTTL)
449+
if err != nil {
450+
return ctrl.Result{}, errors.Wrapf(err, "failed to create new bootstrap token")
451+
}
452+
453+
config.Spec.JoinConfiguration.Discovery.BootstrapToken.Token = token
454+
log.V(3).Info("Altering JoinConfiguration.Discovery.BootstrapToken.Token")
455+
456+
// Update the bootstrap data
457+
return r.joinWorker(ctx, scope)
458+
}
459+
435460
func (r *KubeadmConfigReconciler) rotateMachinePoolBootstrapToken(ctx context.Context, config *bootstrapv1.KubeadmConfig, cluster *clusterv1.Cluster, scope *Scope) (ctrl.Result, error) {
436461
log := ctrl.LoggerFrom(ctx)
437462
log.V(2).Info("Config is owned by a MachinePool, checking if token should be rotated")
@@ -447,16 +472,7 @@ func (r *KubeadmConfigReconciler) rotateMachinePoolBootstrapToken(ctx context.Co
447472
}
448473
if shouldRotate {
449474
log.Info("Creating new bootstrap token, the existing one should be rotated")
450-
token, err := createToken(ctx, remoteClient, r.TokenTTL)
451-
if err != nil {
452-
return ctrl.Result{}, errors.Wrapf(err, "failed to create new bootstrap token")
453-
}
454-
455-
config.Spec.JoinConfiguration.Discovery.BootstrapToken.Token = token
456-
log.V(3).Info("Altering JoinConfiguration.Discovery.BootstrapToken.Token")
457-
458-
// update the bootstrap data
459-
return r.joinWorker(ctx, scope)
475+
return r.recreateBootstrapToken(ctx, config, scope, remoteClient)
460476
}
461477
return ctrl.Result{
462478
RequeueAfter: r.tokenCheckRefreshOrRotationInterval(),

‎bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller_test.go‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1442,6 +1442,155 @@ func TestBootstrapTokenRotationMachinePool(t *testing.T) {
14421442
g.Expect(foundNew).To(BeTrue())
14431443
}
14441444

1445+
func TestBootstrapTokenRefreshIfTokenSecretCleaned(t *testing.T) {
1446+
t.Run("should not recreate the token for Machines", func(t *testing.T) {
1447+
g := NewWithT(t)
1448+
1449+
cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build()
1450+
cluster.Status.InfrastructureReady = true
1451+
conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition)
1452+
cluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{Host: "100.105.150.1", Port: 6443}
1453+
1454+
controlPlaneInitMachine := newControlPlaneMachine(cluster, "control-plane-init-machine")
1455+
initConfig := newControlPlaneInitKubeadmConfig(controlPlaneInitMachine.Namespace, "control-plane-init-config")
1456+
1457+
addKubeadmConfigToMachine(initConfig, controlPlaneInitMachine)
1458+
1459+
workerMachine := newWorkerMachineForCluster(cluster)
1460+
workerJoinConfig := newWorkerJoinKubeadmConfig(metav1.NamespaceDefault, "worker-join-cfg")
1461+
addKubeadmConfigToMachine(workerJoinConfig, workerMachine)
1462+
objects := []client.Object{
1463+
cluster,
1464+
workerMachine,
1465+
workerJoinConfig,
1466+
}
1467+
1468+
objects = append(objects, createSecrets(t, cluster, initConfig)...)
1469+
myclient := fake.NewClientBuilder().WithObjects(objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}).Build()
1470+
remoteClient := fake.NewClientBuilder().Build()
1471+
k := &KubeadmConfigReconciler{
1472+
Client: myclient,
1473+
SecretCachingClient: myclient,
1474+
KubeadmInitLock: &myInitLocker{},
1475+
TokenTTL: DefaultTokenTTL,
1476+
ClusterCache: clustercache.NewFakeClusterCache(remoteClient, client.ObjectKey{Name: cluster.Name, Namespace: cluster.Namespace}),
1477+
}
1478+
request := ctrl.Request{
1479+
NamespacedName: client.ObjectKey{
1480+
Namespace: metav1.NamespaceDefault,
1481+
Name: "worker-join-cfg",
1482+
},
1483+
}
1484+
result, err := k.Reconcile(ctx, request)
1485+
g.Expect(err).ToNot(HaveOccurred())
1486+
g.Expect(result.RequeueAfter).To(Equal(k.TokenTTL / 3))
1487+
1488+
cfg, err := getKubeadmConfig(myclient, "worker-join-cfg", metav1.NamespaceDefault)
1489+
g.Expect(err).ToNot(HaveOccurred())
1490+
g.Expect(cfg.Status.Ready).To(BeTrue())
1491+
g.Expect(cfg.Status.DataSecretName).NotTo(BeNil())
1492+
g.Expect(cfg.Status.ObservedGeneration).NotTo(BeNil())
1493+
g.Expect(cfg.Spec.JoinConfiguration.Discovery.BootstrapToken.Token).ToNot(BeEmpty())
1494+
firstToken := cfg.Spec.JoinConfiguration.Discovery.BootstrapToken.Token
1495+
1496+
l := &corev1.SecretList{}
1497+
g.Expect(remoteClient.List(ctx, l, client.ListOption(client.InNamespace(metav1.NamespaceSystem)))).To(Succeed())
1498+
g.Expect(l.Items).To(HaveLen(1))
1499+
1500+
t.Log("Token should not get recreated for single Machine since it will not use the new token if spec.bootstrap.dataSecretName was already set")
1501+
1502+
// Simulate token cleaner of Kubernetes having deleted the token secret
1503+
err = remoteClient.Delete(ctx, &l.Items[0])
1504+
g.Expect(err).ToNot(HaveOccurred())
1505+
1506+
result, err = k.Reconcile(ctx, request)
1507+
g.Expect(err).To(HaveOccurred())
1508+
g.Expect(err.Error()).To(ContainSubstring("failed to get bootstrap token secret in order to refresh it"))
1509+
// New token should not have been created
1510+
cfg, err = getKubeadmConfig(myclient, "worker-join-cfg", metav1.NamespaceDefault)
1511+
g.Expect(err).ToNot(HaveOccurred())
1512+
g.Expect(cfg.Spec.JoinConfiguration.Discovery.BootstrapToken.Token).To(Equal(firstToken))
1513+
1514+
l = &corev1.SecretList{}
1515+
g.Expect(remoteClient.List(ctx, l, client.ListOption(client.InNamespace(metav1.NamespaceSystem)))).To(Succeed())
1516+
g.Expect(l.Items).To(BeEmpty())
1517+
})
1518+
t.Run("should recreate the token for MachinePools", func(t *testing.T) {
1519+
_ = feature.MutableGates.Set("MachinePool=true")
1520+
g := NewWithT(t)
1521+
1522+
cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build()
1523+
cluster.Status.InfrastructureReady = true
1524+
conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition)
1525+
cluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{Host: "100.105.150.1", Port: 6443}
1526+
1527+
controlPlaneInitMachine := newControlPlaneMachine(cluster, "control-plane-init-machine")
1528+
initConfig := newControlPlaneInitKubeadmConfig(controlPlaneInitMachine.Namespace, "control-plane-init-config")
1529+
1530+
addKubeadmConfigToMachine(initConfig, controlPlaneInitMachine)
1531+
1532+
workerMachinePool := newWorkerMachinePoolForCluster(cluster)
1533+
workerJoinConfig := newWorkerJoinKubeadmConfig(workerMachinePool.Namespace, "workerpool-join-cfg")
1534+
addKubeadmConfigToMachinePool(workerJoinConfig, workerMachinePool)
1535+
objects := []client.Object{
1536+
cluster,
1537+
workerMachinePool,
1538+
workerJoinConfig,
1539+
}
1540+
1541+
objects = append(objects, createSecrets(t, cluster, initConfig)...)
1542+
myclient := fake.NewClientBuilder().WithObjects(objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}, &expv1.MachinePool{}).Build()
1543+
remoteClient := fake.NewClientBuilder().Build()
1544+
k := &KubeadmConfigReconciler{
1545+
Client: myclient,
1546+
SecretCachingClient: myclient,
1547+
KubeadmInitLock: &myInitLocker{},
1548+
TokenTTL: DefaultTokenTTL,
1549+
ClusterCache: clustercache.NewFakeClusterCache(remoteClient, client.ObjectKey{Name: cluster.Name, Namespace: cluster.Namespace}),
1550+
}
1551+
request := ctrl.Request{
1552+
NamespacedName: client.ObjectKey{
1553+
Namespace: metav1.NamespaceDefault,
1554+
Name: "workerpool-join-cfg",
1555+
},
1556+
}
1557+
result, err := k.Reconcile(ctx, request)
1558+
g.Expect(err).ToNot(HaveOccurred())
1559+
g.Expect(result.RequeueAfter).To(Equal(k.TokenTTL / 3))
1560+
1561+
cfg, err := getKubeadmConfig(myclient, "workerpool-join-cfg", metav1.NamespaceDefault)
1562+
g.Expect(err).ToNot(HaveOccurred())
1563+
g.Expect(cfg.Status.Ready).To(BeTrue())
1564+
g.Expect(cfg.Status.DataSecretName).NotTo(BeNil())
1565+
g.Expect(cfg.Status.ObservedGeneration).NotTo(BeNil())
1566+
g.Expect(cfg.Spec.JoinConfiguration.Discovery.BootstrapToken.Token).ToNot(BeEmpty())
1567+
firstToken := cfg.Spec.JoinConfiguration.Discovery.BootstrapToken.Token
1568+
1569+
l := &corev1.SecretList{}
1570+
g.Expect(remoteClient.List(ctx, l, client.ListOption(client.InNamespace(metav1.NamespaceSystem)))).To(Succeed())
1571+
g.Expect(l.Items).To(HaveLen(1))
1572+
1573+
t.Log("Ensure that the token gets recreated if it was cleaned up by Kubernetes (e.g. on expiry)")
1574+
1575+
// Simulate token cleaner of Kubernetes having deleted the token secret
1576+
err = remoteClient.Delete(ctx, &l.Items[0])
1577+
g.Expect(err).ToNot(HaveOccurred())
1578+
1579+
result, err = k.Reconcile(ctx, request)
1580+
g.Expect(err).ToNot(HaveOccurred())
1581+
g.Expect(result.RequeueAfter).To(Equal(k.TokenTTL / 3))
1582+
// New token should have been created
1583+
cfg, err = getKubeadmConfig(myclient, "workerpool-join-cfg", metav1.NamespaceDefault)
1584+
g.Expect(err).ToNot(HaveOccurred())
1585+
g.Expect(cfg.Spec.JoinConfiguration.Discovery.BootstrapToken.Token).ToNot(BeEmpty())
1586+
g.Expect(cfg.Spec.JoinConfiguration.Discovery.BootstrapToken.Token).ToNot(Equal(firstToken))
1587+
1588+
l = &corev1.SecretList{}
1589+
g.Expect(remoteClient.List(ctx, l, client.ListOption(client.InNamespace(metav1.NamespaceSystem)))).To(Succeed())
1590+
g.Expect(l.Items).To(HaveLen(1))
1591+
})
1592+
}
1593+
14451594
// Ensure the discovery portion of the JoinConfiguration gets generated correctly.
14461595
func TestKubeadmConfigReconciler_Reconcile_DiscoveryReconcileBehaviors(t *testing.T) {
14471596
caHash := []string{"...."}

0 commit comments

Comments
 (0)