Merge branch 'k8s-pod-event-on-failure'

Add warning events on failure with k8s executor

Closes https://gitlab.com/gitlab-org/gitlab-runner/-/issues/31052

See merge request https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/4211

Merged-by: Romuald Atchadé <[email protected]>
Approved-by: Axel von Bertoldi <[email protected]>
Reviewed-by: Axel von Bertoldi <[email protected]>

Author: Romuald Atchadé

executors/kubernetes/kubernetes.go

@@ -84,6 +84,8 @@ const (
 	k8sResourcesNameSuffixLength = 8
 	k8sResourcesNameMaxLength    = 63
 
+	k8sEventWarningType = "Warning"
+
 	// errorDialingBackendMessage is an error prefix that is encountered when
 	// connectivity to a Pod fails. This can happen for a number of reasons,
 	// such as the Pod or Node still being configured.
@@ -448,6 +450,10 @@ func (s *executor) Run(cmd common.ExecutorCommand) error {
 			err = s.runWithAttach(cmd)
 		}
 
+		if err != nil {
+			s.logPodWarningEvents(k8sEventWarningType)
+		}
+
 		var imagePullErr *pull.ImagePullError
 		if errors.As(err, &imagePullErr) {
 			if s.pullManager.UpdatePolicyForImage(attempt, imagePullErr) {
@@ -460,6 +466,41 @@ func (s *executor) Run(cmd common.ExecutorCommand) error {
 	}
 }
 
+func (s *executor) logPodWarningEvents(eventType string) {
+	if s.pod == nil {
+		return
+	}
+
+	var eventList *api.EventList
+	r := retry.WithBuildLog(
+		&retryableKubeAPICall{
+			maxTries: defaultTries,
+			fn: func() error {
+				// TODO: handle the context properly with https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27932
+				el, err := s.kubeClient.CoreV1().Events(s.pod.Namespace).
+					List(context.Background(), metav1.ListOptions{
+						FieldSelector: fmt.Sprintf("involvedObject.name=%s,type=%s", s.pod.Name, eventType),
+					})
+				if err == nil {
+					eventList = el
+				}
+				return err
+			},
+		},
+		&s.BuildLogger,
+	)
+	retryable := retry.NewWithBackoffDuration(r, defaultRetryMinBackoff, defaultRetryMaxBackoff)
+	err := retryable.Run()
+	if err != nil {
+		s.Errorln(fmt.Sprintf("Error retrieving events list: %s", err.Error()))
+		return
+	}
+
+	for _, event := range eventList.Items {
+		s.Warningln(fmt.Sprintf("Event retrieved from the cluster: %s", event.Message))
+	}
+}
+
 func (s *executor) runWithExecLegacy(cmd common.ExecutorCommand) error {
 	if s.pod == nil {
 		err := s.setupCredentials()

executors/kubernetes/kubernetes_integration_test.go

@@ -133,6 +133,7 @@ func TestRunIntegrationTestsWithFeatureFlag(t *testing.T) {
 		"testKubernetesLongLogsFeatureFlag":                       testKubernetesLongLogsFeatureFlag,
 		"testKubernetesHugeScriptAndAfterScriptFeatureFlag":       testKubernetesHugeScriptAndAfterScriptFeatureFlag,
 		"testKubernetesCustomPodSpec":                             testKubernetesCustomPodSpec,
+		"testKubernetesClusterWarningEvent":                       testKubernetesClusterWarningEvent,
 	}
 
 	featureFlags := []string{
@@ -1565,6 +1566,47 @@ func testKubernetesWaitResources(t *testing.T, featureFlagName string, featureFl
 	}
 }
 
+func testKubernetesClusterWarningEvent(t *testing.T, featureFlagName string, featureFlagValue bool) {
+	helpers.SkipIntegrationTests(t, "kubectl", "cluster-info")
+
+	tests := map[string]struct {
+		image    string
+		verifyFn func(*testing.T, string, error)
+	}{
+		"invalid image": {
+			image: "alpine333",
+			verifyFn: func(t *testing.T, out string, err error) {
+				assert.Error(t, err)
+				assert.Contains(
+					t,
+					out,
+					"WARNING: Event retrieved from the cluster: Failed to pull image \"alpine333\": rpc error:",
+				)
+				assert.Contains(t, out, "WARNING: Event retrieved from the cluster: Error: ErrImagePull")
+				assert.Contains(t, out, "WARNING: Event retrieved from the cluster: Error: ImagePullBackOff")
+			},
+		},
+	}
+
+	for tn, tc := range tests {
+		t.Run(tn, func(t *testing.T) {
+			build := getTestBuild(t, func() (common.JobResponse, error) {
+				jobResponse, err := common.GetRemoteBuildResponse(
+					"echo Hello World",
+				)
+				require.NoError(t, err)
+
+				return jobResponse, nil
+			})
+			build.Runner.Kubernetes.Image = tc.image
+			buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue)
+
+			out, err := buildtest.RunBuildReturningOutput(t, build)
+			tc.verifyFn(t, out, err)
+		})
+	}
+}
+
 // TestLogDeletionAttach tests the outcome when the log files are all deleted
 func TestLogDeletionAttach(t *testing.T) {
 	helpers.SkipIntegrationTests(t, "kubectl", "cluster-info")