fix: concurrency issue with workflow cleanup executor (#1495)

Author: csviri

operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/workflow/WorkflowCleanupExecutor.java

@@ -104,16 +104,20 @@ public void run() {
           ((Deleter<P>) dependentResourceNode.getDependentResource()).delete(primary, context);
           deleteCalled.add(dependentResourceNode);
         }
-        alreadyVisited.add(dependentResourceNode);
         boolean deletePostConditionMet =
             deletePostCondition.map(c -> c.isMet(primary,
                 dependentResourceNode.getDependentResource().getSecondaryResource(primary)
                     .orElse(null),
                 context)).orElse(true);
+
         if (deletePostConditionMet) {
+          alreadyVisited.add(dependentResourceNode);
           handleDependentCleaned(dependentResourceNode);
         } else {
+          // updating alreadyVisited needs to be the last operation otherwise could lead to a race
+          // condition in handleCleanup condition checks
           postDeleteConditionNotMet.add(dependentResourceNode);
+          alreadyVisited.add(dependentResourceNode);
         }
       } catch (RuntimeException e) {
         handleExceptionInExecutor(dependentResourceNode, e);

operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/workflow/WorkflowReconcileExecutor.java

@@ -217,6 +217,8 @@ public void run() {
           alreadyVisited.add(dependentResourceNode);
           handleDependentDeleted(dependentResourceNode);
         } else {
+          // updating alreadyVisited needs to be the last operation otherwise could lead to a race
+          // condition in handleDelete condition checks
           deletePostConditionNotMet.add(dependentResourceNode);
           alreadyVisited.add(dependentResourceNode);
         }