run tests on loop until failure

Author: ivelichkovich

Makefile

@@ -925,7 +925,7 @@ test-verbose: ## Run unit and integration tests with race detector and with verb
 
 .PHONY: test-junit
 test-junit: $(SETUP_ENVTEST) $(GOTESTSUM) ## Run unit and integration tests with race detector and generate a junit report
-	set +o errexit; (KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test -race -json ./... $(TEST_ARGS); echo $$? > $(ARTIFACTS)/junit.exitcode) | tee $(ARTIFACTS)/junit.stdout
+	set +o errexit; (KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test -count=1 -race -json ./... $(TEST_ARGS); echo $$? > $(ARTIFACTS)/junit.exitcode) | tee $(ARTIFACTS)/junit.stdout
 	$(GOTESTSUM) --junitfile $(ARTIFACTS)/junit.xml --raw-command cat $(ARTIFACTS)/junit.stdout
 	exit $$(cat $(ARTIFACTS)/junit.exitcode)
 	set +o errexit; (KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test -run "^TestFuzzyConversion$$" -json ./... $(TEST_ARGS); echo $$? > $(ARTIFACTS)/junit-fuzz.exitcode) | tee $(ARTIFACTS)/junit-fuzz.stdout

internal/controllers/machine/machine_controller_status.go

@@ -43,6 +43,7 @@ import (
 // Additionally, this func should ensure that the conditions managed by this controller are always set in order to
 // comply with the recommendation in the Kubernetes API guidelines.
 func (r *Reconciler) updateStatus(ctx context.Context, s *scope) {
+	log := ctrl.LoggerFrom(ctx)
 	// Update status from the Bootstrap Config external resource.
 	// Note: some of the status fields derived from the Bootstrap Config are managed in reconcileBootstrap, e.g. status.BootstrapReady, etc.
 	// here we are taking care only of the delta (condition).
@@ -51,12 +52,15 @@ func (r *Reconciler) updateStatus(ctx context.Context, s *scope) {
 	// Update status from the InfraMachine external resource.
 	// Note: some of the status fields derived from the InfraMachine are managed in reconcileInfrastructure, e.g. status.InfrastructureReady, etc.
 	// here we are taking care only of the delta (condition).
+	log.V(6).Info("setting infrastructureReadyCondition")
 	setInfrastructureReadyCondition(ctx, s.machine, s.infraMachine, s.infraMachineIsNotFound)
 
 	// Update status from the Node external resource.
 	// Note: some of the status fields are managed in reconcileNode, e.g. status.NodeRef, etc.
 	// here we are taking care only of the delta (condition).
+	log.V(6).Info("getting clustercache healthcheckstate")
 	healthCheckingState := r.ClusterCache.GetHealthCheckingState(ctx, client.ObjectKeyFromObject(s.cluster))
+	log.V(6).Info("setting nodeHealthAndReadyConditions")
 	setNodeHealthyAndReadyConditions(ctx, s.cluster, s.machine, s.node, s.nodeGetError, healthCheckingState, r.RemoteConditionsGracePeriod)
 
 	// Updates Machine status not observed from Bootstrap Config, InfraMachine or Node (update Machine's own status).

scripts/ci-test.sh

@@ -25,13 +25,17 @@ cd "${REPO_ROOT}" || exit 1
 source "${REPO_ROOT}/hack/ensure-go.sh"
 
 echo "*** Testing Cluster API ***"
-make test-junit
+# temp run on loop to try to catch flake
 
-echo -e "\n*** Testing test/infrastructure folder ***\n"
-make test-infrastructure-junit
+CAPI_TEST_ENV_LOG_LEVEL=10
+while make test-junit; do :; done
 
-echo -e "\n*** Testing Cluster API Runtime SDK test extension ***\n"
-make test-test-extension-junit
 
-echo -e "\n*** Testing Cluster API testing framework ***\n"
-make test-framework-junit
+#echo -e "\n*** Testing test/infrastructure folder ***\n"
+#make test-infrastructure-junit
+#
+#echo -e "\n*** Testing Cluster API Runtime SDK test extension ***\n"
+#make test-test-extension-junit
+#
+#echo -e "\n*** Testing Cluster API testing framework ***\n"
+#make test-framework-junit