Skip to content

Commit 1b8ddbd

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #730 from w21froster/fix-lagging-provider-check-failure
🐛: fix unknown provider errors when using fetchConfigs
2 parents fe7e70d + b98394f commit 1b8ddbd

File tree

8 files changed

+30802
-25
lines changed

8 files changed

+30802
-25
lines changed

‎internal/controller/phases.go‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,18 @@ func (p *phaseReconciler) initializePhaseReconciler(ctx context.Context) (reconc
147147
return reconcile.Result{}, err
148148
}
149149

150+
// Get all custom providers using fetchConfig that aren't the current provider.
151+
customProviders, err := util.GetCustomProviders(ctx, p.ctrlClient, p.provider)
152+
if err != nil {
153+
return reconcile.Result{}, err
154+
}
155+
156+
// Load all custom providers into MemoryReader.
157+
reader, err = loadCustomProviders(customProviders, reader)
158+
if err != nil {
159+
return reconcile.Result{}, err
160+
}
161+
150162
// Load provider's secret and config url.
151163
p.configClient, err = configclient.New(ctx, "", configclient.InjectReader(reader))
152164
if err != nil {
@@ -562,6 +574,28 @@ func getProvider(provider operatorv1.GenericProvider, defaultVersion string) clu
562574
return *clusterctlProvider
563575
}
564576

577+
// loadCustomProviders loads the passed providers list into the clusterctl configuration via the MemoryReader.
578+
func loadCustomProviders(providers []operatorv1.GenericProvider, reader configclient.Reader) (configclient.Reader, error) {
579+
mr, ok := reader.(*configclient.MemoryReader)
580+
if !ok {
581+
return nil, fmt.Errorf("unable to load custom providers, invalid reader passed")
582+
}
583+
584+
for _, provider := range providers {
585+
if provider.GetSpec().FetchConfig.URL == "" {
586+
if _, err := mr.AddProvider(provider.GetName(), util.ClusterctlProviderType(provider), fakeURL); err != nil {
587+
return nil, err
588+
}
589+
} else {
590+
if _, err := mr.AddProvider(provider.GetName(), util.ClusterctlProviderType(provider), provider.GetSpec().FetchConfig.URL); err != nil {
591+
return nil, err
592+
}
593+
}
594+
}
595+
596+
return mr, nil
597+
}
598+
565599
// delete deletes the provider components using clusterctl library.
566600
func (p *phaseReconciler) delete(ctx context.Context) (reconcile.Result, error) {
567601
log := ctrl.LoggerFrom(ctx)

‎test/e2e/air_gapped_test.go‎

Lines changed: 263 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -34,39 +34,91 @@ import (
3434
"sigs.k8s.io/yaml"
3535
)
3636

37-
var _ = Describe("Install Core Provider in an air-gapped environment", func() {
38-
It("should successfully create config maps with Core Provider manifests", func() {
37+
var _ = Describe("Install Controlplane, Core, Bootstrap Providers in an air-gapped environment", func() {
38+
It("should successfully create config maps with Controlplane, Core, and Bootstrap Provider manifests", func() {
3939
// Ensure that there are no Cluster API installed
4040
deleteClusterAPICRDs(bootstrapClusterProxy)
4141

4242
bootstrapCluster := bootstrapClusterProxy.GetClient()
4343
configMaps := []corev1.ConfigMap{}
44+
configMapFiles := []string{
45+
"core-cluster-api-v1.7.7.yaml",
46+
"core-cluster-api-v1.8.0.yaml",
47+
"bootstrap-kubeadm-v1.7.7.yaml",
48+
"bootstrap-kubeadm-v1.8.0.yaml",
49+
"controlplane-kubeadm-v1.7.7.yaml",
50+
"controlplane-kubeadm-v1.8.0.yaml",
51+
}
4452

45-
for _, fileName := range []string{"core-cluster-api-v1.7.7.yaml", "core-cluster-api-v1.8.0.yaml"} {
46-
coreProviderComponents, err := os.ReadFile(customManifestsFolder + fileName)
47-
Expect(err).ToNot(HaveOccurred(), "Failed to read the core provider manifests file")
53+
for _, fileName := range configMapFiles {
54+
providerComponents, err := os.ReadFile(customManifestsFolder + fileName)
55+
Expect(err).ToNot(HaveOccurred(), "Failed to read the provider manifests file")
4856

4957
var configMap corev1.ConfigMap
5058

51-
Expect(yaml.Unmarshal(coreProviderComponents, &configMap)).To(Succeed())
59+
Expect(yaml.Unmarshal(providerComponents, &configMap)).To(Succeed())
5260

5361
configMaps = append(configMaps, configMap)
5462
}
5563

56-
By("Creating capi-system namespace")
57-
namespace := &corev1.Namespace{
58-
ObjectMeta: metav1.ObjectMeta{
59-
Name: capiSystemNamespace,
60-
},
64+
By("Creating provider namespaces")
65+
for _, namespaceName := range []string{cabpkSystemNamespace, cacpkSystemNamespace, capiSystemNamespace} {
66+
namespace := &corev1.Namespace{
67+
ObjectMeta: metav1.ObjectMeta{
68+
Name: namespaceName,
69+
},
70+
}
71+
Expect(bootstrapCluster.Create(ctx, namespace)).To(Succeed())
6172
}
62-
Expect(bootstrapCluster.Create(ctx, namespace)).To(Succeed())
6373

64-
By("Applying core provider manifests to the cluster")
74+
By("Applying provider manifests to the cluster")
6575
for _, cm := range configMaps {
6676
Expect(bootstrapCluster.Create(ctx, &cm)).To(Succeed())
6777
}
6878
})
6979

80+
It("should successfully create a BootstrapProvider from a config map", func() {
81+
bootstrapCluster := bootstrapClusterProxy.GetClient()
82+
bootstrapProvider := &operatorv1.BootstrapProvider{
83+
ObjectMeta: metav1.ObjectMeta{
84+
Name: customProviderName,
85+
Namespace: cabpkSystemNamespace,
86+
},
87+
Spec: operatorv1.BootstrapProviderSpec{
88+
ProviderSpec: operatorv1.ProviderSpec{
89+
FetchConfig: &operatorv1.FetchConfiguration{
90+
Selector: &metav1.LabelSelector{
91+
MatchLabels: map[string]string{
92+
"provider.cluster.x-k8s.io/name": "kubeadm",
93+
"provider.cluster.x-k8s.io/type": "bootstrap",
94+
"provider.cluster.x-k8s.io/version": "v1.7.7",
95+
},
96+
},
97+
},
98+
Version: previousCAPIVersion,
99+
},
100+
},
101+
}
102+
103+
Expect(bootstrapCluster.Create(ctx, bootstrapProvider)).To(Succeed())
104+
105+
By("Waiting for the bootstrap provider deployment to be ready")
106+
framework.WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{
107+
Getter: bootstrapClusterProxy.GetClient(),
108+
Deployment: &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: bootstrapProviderDeploymentName, Namespace: cabpkSystemNamespace}},
109+
}, e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...)
110+
111+
By("Waiting for bootstrap provider to be ready")
112+
WaitFor(ctx, For(bootstrapProvider).In(bootstrapCluster).ToSatisfy(
113+
HaveStatusCondition(&bootstrapProvider.Status.Conditions, operatorv1.ProviderInstalledCondition),
114+
), e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...)
115+
116+
By("Waiting for status.InstalledVersion to be set")
117+
WaitFor(ctx, For(bootstrapProvider).In(bootstrapCluster).ToSatisfy(func() bool {
118+
return ptr.Equal(bootstrapProvider.Status.InstalledVersion, ptr.To(bootstrapProvider.Spec.Version))
119+
}), e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...)
120+
})
121+
70122
It("should successfully create a CoreProvider from a config map", func() {
71123
bootstrapCluster := bootstrapClusterProxy.GetClient()
72124
coreProvider := &operatorv1.CoreProvider{
@@ -79,8 +131,9 @@ var _ = Describe("Install Core Provider in an air-gapped environment", func() {
79131
FetchConfig: &operatorv1.FetchConfiguration{
80132
Selector: &metav1.LabelSelector{
81133
MatchLabels: map[string]string{
82-
"provider.cluster.x-k8s.io/name": "cluster-api",
83-
"provider.cluster.x-k8s.io/type": "core",
134+
"provider.cluster.x-k8s.io/name": "cluster-api",
135+
"provider.cluster.x-k8s.io/type": "core",
136+
"provider.cluster.x-k8s.io/version": "v1.7.7",
84137
},
85138
},
86139
},
@@ -108,15 +161,90 @@ var _ = Describe("Install Core Provider in an air-gapped environment", func() {
108161
}), e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...)
109162
})
110163

111-
It("should successfully upgrade a CoreProvider (v1.7.7 -> latest)", func() {
164+
It("should successfully create a ControlPlaneProvider from a config map", func() {
165+
bootstrapCluster := bootstrapClusterProxy.GetClient()
166+
controlPlaneProvider := &operatorv1.ControlPlaneProvider{
167+
ObjectMeta: metav1.ObjectMeta{
168+
Name: customProviderName,
169+
Namespace: cacpkSystemNamespace,
170+
},
171+
Spec: operatorv1.ControlPlaneProviderSpec{
172+
ProviderSpec: operatorv1.ProviderSpec{
173+
FetchConfig: &operatorv1.FetchConfiguration{
174+
Selector: &metav1.LabelSelector{
175+
MatchLabels: map[string]string{
176+
"provider.cluster.x-k8s.io/name": "kubeadm",
177+
"provider.cluster.x-k8s.io/type": "controlplane",
178+
"provider.cluster.x-k8s.io/version": "v1.7.7",
179+
},
180+
},
181+
},
182+
Version: previousCAPIVersion,
183+
},
184+
},
185+
}
186+
187+
Expect(bootstrapCluster.Create(ctx, controlPlaneProvider)).To(Succeed())
188+
189+
By("Waiting for the controlplane provider deployment to be ready")
190+
framework.WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{
191+
Getter: bootstrapClusterProxy.GetClient(),
192+
Deployment: &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: cpProviderDeploymentName, Namespace: cacpkSystemNamespace}},
193+
}, e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...)
194+
195+
By("Waiting for controlplane provider to be ready")
196+
WaitFor(ctx, For(controlPlaneProvider).In(bootstrapCluster).ToSatisfy(
197+
HaveStatusCondition(&controlPlaneProvider.Status.Conditions, operatorv1.ProviderInstalledCondition),
198+
), e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...)
199+
200+
By("Waiting for status.InstalledVersion to be set")
201+
WaitFor(ctx, For(controlPlaneProvider).In(bootstrapCluster).ToSatisfy(func() bool {
202+
return ptr.Equal(controlPlaneProvider.Status.InstalledVersion, ptr.To(controlPlaneProvider.Spec.Version))
203+
}), e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...)
204+
})
205+
206+
It("should successfully upgrade BootstrapProvider, ControlPlaneProvider, CoreProvider (v1.7.7 -> v1.8.0)", func() {
112207
bootstrapCluster := bootstrapClusterProxy.GetClient()
208+
209+
bootstrapProvider := &operatorv1.BootstrapProvider{}
113210
coreProvider := &operatorv1.CoreProvider{}
114-
key := client.ObjectKey{Namespace: capiSystemNamespace, Name: coreProviderName}
115-
Expect(bootstrapCluster.Get(ctx, key, coreProvider)).To(Succeed())
211+
controlPlaneProvider := &operatorv1.ControlPlaneProvider{}
212+
213+
bootstrapKey := client.ObjectKey{Namespace: cabpkSystemNamespace, Name: customProviderName}
214+
Expect(bootstrapCluster.Get(ctx, bootstrapKey, bootstrapProvider)).To(Succeed())
215+
216+
coreKey := client.ObjectKey{Namespace: capiSystemNamespace, Name: coreProviderName}
217+
Expect(bootstrapCluster.Get(ctx, coreKey, coreProvider)).To(Succeed())
116218

117-
coreProvider.Spec.Version = ""
219+
cpKey := client.ObjectKey{Namespace: cacpkSystemNamespace, Name: customProviderName}
220+
Expect(bootstrapCluster.Get(ctx, cpKey, controlPlaneProvider)).To(Succeed())
118221

222+
bootstrapProvider.Spec.Version = nextCAPIVersion
223+
bootstrapProvider.Spec.FetchConfig.Selector.MatchLabels["provider.cluster.x-k8s.io/version"] = nextCAPIVersion
224+
coreProvider.Spec.Version = nextCAPIVersion
225+
coreProvider.Spec.FetchConfig.Selector.MatchLabels["provider.cluster.x-k8s.io/version"] = nextCAPIVersion
226+
controlPlaneProvider.Spec.Version = nextCAPIVersion
227+
controlPlaneProvider.Spec.FetchConfig.Selector.MatchLabels["provider.cluster.x-k8s.io/version"] = nextCAPIVersion
228+
229+
Expect(bootstrapCluster.Update(ctx, bootstrapProvider)).To(Succeed())
119230
Expect(bootstrapCluster.Update(ctx, coreProvider)).To(Succeed())
231+
Expect(bootstrapCluster.Update(ctx, controlPlaneProvider)).To(Succeed())
232+
233+
By("Waiting for the bootstrap provider deployment to be ready")
234+
framework.WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{
235+
Getter: bootstrapClusterProxy.GetClient(),
236+
Deployment: &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: bootstrapProviderDeploymentName, Namespace: cabpkSystemNamespace}},
237+
}, e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...)
238+
239+
By("Waiting for bootstrap provider to be ready")
240+
WaitFor(ctx, For(bootstrapProvider).In(bootstrapCluster).ToSatisfy(
241+
HaveStatusCondition(&bootstrapProvider.Status.Conditions, operatorv1.ProviderInstalledCondition),
242+
), e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...)
243+
244+
By("Waiting for bootstrap provider status.InstalledVersion to be set")
245+
WaitFor(ctx, For(bootstrapProvider).In(bootstrapCluster).ToSatisfy(func() bool {
246+
return ptr.Equal(bootstrapProvider.Status.InstalledVersion, ptr.To(bootstrapProvider.Spec.Version))
247+
}), e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...)
120248

121249
By("Waiting for the core provider deployment to be ready")
122250
framework.WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{
@@ -133,6 +261,72 @@ var _ = Describe("Install Core Provider in an air-gapped environment", func() {
133261
WaitFor(ctx, For(coreProvider).In(bootstrapCluster).ToSatisfy(func() bool {
134262
return ptr.Equal(coreProvider.Status.InstalledVersion, ptr.To(coreProvider.Spec.Version))
135263
}), e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...)
264+
265+
By("Waiting for the controlplane provider deployment to be ready")
266+
framework.WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{
267+
Getter: bootstrapClusterProxy.GetClient(),
268+
Deployment: &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: cpProviderDeploymentName, Namespace: cacpkSystemNamespace}},
269+
}, e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...)
270+
271+
By("Waiting for controlplane provider to be ready")
272+
WaitFor(ctx, For(controlPlaneProvider).In(bootstrapCluster).ToSatisfy(
273+
HaveStatusCondition(&coreProvider.Status.Conditions, operatorv1.ProviderInstalledCondition),
274+
), e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...)
275+
276+
By("Waiting for status.InstalledVersion to be set")
277+
WaitFor(ctx, For(controlPlaneProvider).In(bootstrapCluster).ToSatisfy(func() bool {
278+
return ptr.Equal(controlPlaneProvider.Status.InstalledVersion, ptr.To(controlPlaneProvider.Spec.Version))
279+
}), e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...)
280+
})
281+
282+
It("should successfully delete a BootstrapProvider", func() {
283+
bootstrapCluster := bootstrapClusterProxy.GetClient()
284+
bootstrapProvider := &operatorv1.BootstrapProvider{ObjectMeta: metav1.ObjectMeta{
285+
Name: customProviderName,
286+
Namespace: cabpkSystemNamespace,
287+
}}
288+
289+
Expect(bootstrapCluster.Delete(ctx, bootstrapProvider)).To(Succeed())
290+
291+
By("Waiting for the bootstrap provider deployment to be deleted")
292+
WaitForDelete(ctx, For(&appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{
293+
Name: bootstrapProviderDeploymentName,
294+
Namespace: cabpkSystemNamespace,
295+
}}).In(bootstrapCluster), e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...)
296+
297+
By("Waiting for the bootstrap provider object to be deleted")
298+
WaitForDelete(
299+
ctx, For(bootstrapProvider).In(bootstrapCluster),
300+
e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...)
301+
})
302+
303+
It("should successfully delete config maps with Bootstrap Provider manifests", func() {
304+
bootstrapCluster := bootstrapClusterProxy.GetClient()
305+
configMaps := []corev1.ConfigMap{}
306+
307+
for _, fileName := range []string{"bootstrap-kubeadm-v1.7.7.yaml", "bootstrap-kubeadm-v1.8.0.yaml"} {
308+
bootstrapProviderComponents, err := os.ReadFile(customManifestsFolder + fileName)
309+
Expect(err).ToNot(HaveOccurred(), "Failed to read the bootstrap provider manifests file")
310+
311+
var configMap corev1.ConfigMap
312+
313+
Expect(yaml.Unmarshal(bootstrapProviderComponents, &configMap)).To(Succeed())
314+
315+
configMaps = append(configMaps, configMap)
316+
}
317+
318+
By("Deleting config maps with bootstrap provider manifests")
319+
for _, cm := range configMaps {
320+
Expect(bootstrapCluster.Delete(ctx, &cm)).To(Succeed())
321+
}
322+
323+
By("Deleting capkb-system namespace")
324+
namespace := &corev1.Namespace{
325+
ObjectMeta: metav1.ObjectMeta{
326+
Name: cabpkSystemNamespace,
327+
},
328+
}
329+
Expect(bootstrapCluster.Delete(ctx, namespace)).To(Succeed())
136330
})
137331

138332
It("should successfully delete a CoreProvider", func() {
@@ -184,4 +378,54 @@ var _ = Describe("Install Core Provider in an air-gapped environment", func() {
184378
}
185379
Expect(bootstrapCluster.Delete(ctx, namespace)).To(Succeed())
186380
})
381+
382+
It("should successfully delete a ControlPlaneProvider", func() {
383+
bootstrapCluster := bootstrapClusterProxy.GetClient()
384+
ControlPlaneProvider := &operatorv1.ControlPlaneProvider{ObjectMeta: metav1.ObjectMeta{
385+
Name: customProviderName,
386+
Namespace: cacpkSystemNamespace,
387+
}}
388+
389+
Expect(bootstrapCluster.Delete(ctx, ControlPlaneProvider)).To(Succeed())
390+
391+
By("Waiting for the controlplane provider deployment to be deleted")
392+
WaitForDelete(ctx, For(&appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{
393+
Name: cpProviderDeploymentName,
394+
Namespace: cacpkSystemNamespace,
395+
}}).In(bootstrapCluster), e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...)
396+
397+
By("Waiting for the controlplane provider object to be deleted")
398+
WaitForDelete(
399+
ctx, For(ControlPlaneProvider).In(bootstrapCluster),
400+
e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...)
401+
})
402+
403+
It("should successfully delete config maps with ControlPlane Provider manifests", func() {
404+
bootstrapCluster := bootstrapClusterProxy.GetClient()
405+
configMaps := []corev1.ConfigMap{}
406+
407+
for _, fileName := range []string{"controlplane-kubeadm-v1.7.7.yaml", "controlplane-kubeadm-v1.8.0.yaml"} {
408+
controlPlaneProviderComponents, err := os.ReadFile(customManifestsFolder + fileName)
409+
Expect(err).ToNot(HaveOccurred(), "Failed to read the controlplane provider manifests file")
410+
411+
var configMap corev1.ConfigMap
412+
413+
Expect(yaml.Unmarshal(controlPlaneProviderComponents, &configMap)).To(Succeed())
414+
415+
configMaps = append(configMaps, configMap)
416+
}
417+
418+
By("Deleting config maps with controlplane provider manifests")
419+
for _, cm := range configMaps {
420+
Expect(bootstrapCluster.Delete(ctx, &cm)).To(Succeed())
421+
}
422+
423+
By("Deleting capkcp-system namespace")
424+
namespace := &corev1.Namespace{
425+
ObjectMeta: metav1.ObjectMeta{
426+
Name: cacpkSystemNamespace,
427+
},
428+
}
429+
Expect(bootstrapCluster.Delete(ctx, namespace)).To(Succeed())
430+
})
187431
})

0 commit comments

Comments
 (0)