Skip updates on Application with deletionTimestamp in the past

metadata.deletionTimestamp is set when a resource is deleted with
propagationPolicy: orphan or propagationPolicy: foreground. The resource
may be updated after this, for example finalizers may be set or removed.
If the application resource is in the process of being deleted (i.e. has
deletionTimestamp set and in the past), skip deploy deploy to avoid
interfering with the deletion process done by the garbage collector.

fiaas_deploy_daemon/crd/watcher.py

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
+import datetime
 import logging
 from queue import Queue
 from typing import Union
@@ -90,6 +90,22 @@ def _skip_status_event(self, application: FiaasApplication):
             return True
         return False
 
+    def _skip_update_of_deleted_application(self, application: FiaasApplication):
+        # metadata.deletionTimestamp is set when a resource is deleted with propagationPolicy: orphan or
+        # propagationPolicy: foreground. The resource may be updated after this, for example finalizers may be set or
+        # removed. If the application resource is in the process of being deleted, deploy should be skipped otherwise
+        # it may interfere with the deletion process done by the garbage collector.
+        deletion_timestamp = application.metadata.deletionTimestamp
+        if isinstance(deletion_timestamp, datetime.datetime) and deletion_timestamp <= datetime.datetime.now(
+            tz=datetime.timezone.utc
+        ):
+            LOG.warning(
+                "Skipping update watch event for app %s; it was marked for deletion at %s",
+                application.spec.application,
+                deletion_timestamp,
+            )
+            return True
+
     def _deploy(self, application: FiaasApplication):
         app_name = application.spec.application
         LOG.debug("Deploying %s", app_name)
@@ -100,6 +116,8 @@ def _deploy(self, application: FiaasApplication):
             raise ValueError("The Application {} is missing the 'fiaas/deployment_id' label".format(app_name))
         if self._skip_status_event(application):
             return
+        if self._skip_update_of_deleted_application(application):
+            return
         if self._already_deployed(app_name, application.metadata.namespace, deployment_id):
             LOG.debug("Have already deployed %s for app %s", deployment_id, app_name)
             return

tests/fiaas_deploy_daemon/crd/test_crd_watcher.py

@@ -16,6 +16,7 @@
 # limitations under the License.
 
 
+import copy
 from queue import Queue
 
 from unittest import mock
@@ -311,3 +312,12 @@ def test_deploy_save_status(self, crd_watcher, deploy_queue, watcher, status_get
         assert deploy_queue.qsize() == 0
         crd_watcher._watch(None)
         assert deploy_queue.qsize() == 0
+
+    def test_deploy_skip_deleted_app(self, crd_watcher, deploy_queue, watcher, status_get):
+        event = copy.deepcopy(MODIFIED_EVENT)
+        event['object']['metadata']['deletionTimestamp'] = '2000-01-01T00:00:00Z'
+        watcher.watch.return_value = [WatchEvent(event, FiaasApplication)]
+
+        assert deploy_queue.qsize() == 0
+        crd_watcher._watch(None)
+        assert deploy_queue.qsize() == 0