handling git-repo connection-issues (fix #83)

Author: superstes

src/ansibleguy-webui/aw/execute/play.py

@@ -9,7 +9,7 @@
 from aw.execute.repository import ExecuteRepository
 from aw.execute.alert import Alert
 from aw.utils.util import datetime_w_tz, is_null, timed_lru_cache  # get_ansible_versions
-from aw.utils.handlers import AnsibleConfigError
+from aw.utils.handlers import AnsibleConfigError, AnsibleRepositoryError
 from aw.utils.debug import log
 
 
@@ -63,7 +63,10 @@ def _cancel_job() -> bool:
         runner_cleanup(execution=execution, path_run=path_run, exec_repo=exec_repo)
         Alert(job=job, execution=execution).go()
 
-    except (OSError, AnsibleConfigError, ValueError, AttributeError, IndexError, KeyError) as err:
+    except (
+            AnsibleConfigError, AnsibleRepositoryError,
+            OSError, ValueError, AttributeError, IndexError, KeyError,
+    ) as err:
         tb = traceback.format_exc(limit=1024)
         failure(
             execution=execution, exec_repo=exec_repo, path_run=path_run, result=result,

src/ansibleguy-webui/aw/execute/repository.py

@@ -121,7 +121,10 @@ def get_path_run_repo(self) -> Path:
         return self.path_run / '.repository'
 
     def _git_env(self) -> dict:
-        env = {'GIT_SSH_COMMAND': 'ssh'}
+        env = {
+            'GIT_SSH_COMMAND': f'ssh -o ConnectTimeout=10',
+            'GIT_HTTP_CONNECT_TIMEOUT': '10',
+        }
         if self.repository.git_origin.find(' -p') != -1:
             try:
                 self.repository.git_origin, port = self.repository.git_origin.split(' -p')
@@ -182,6 +185,7 @@ def _repo_process(self, cmd: str, env: dict):
         if self.path_repo is None:
             self.path_repo = self.get_path_repo()
 
+        cmd = f'timeout {self.repository.git_timeout} {cmd}'
         result = process(cmd=cmd, cwd=self.path_repo, env=env, shell=True, timeout_sec=REPO_CLONE_TIMEOUT)
         self._log_file_write(f"COMMAND: {cmd}\n{result['stdout']}")
         if result['rc'] != 0:

src/ansibleguy-webui/aw/execute/threader.py

@@ -12,7 +12,7 @@
 from aw.utils.handlers import AnsibleConfigError, AnsibleRepositoryError
 from aw.utils.util import get_next_cron_execution_sec, get_next_cron_execution_str, is_set
 from aw.execute.util import update_status
-from aw.model.base import JOB_EXEC_STATUS_ACTIVE, JOB_EXEC_STATUS_FAILED
+from aw.model.base import JOB_EXEC_STATI_ACTIVE, JOB_EXEC_STATUS_FAILED, JOB_EXEC_STATUS_RETRY
 
 
 class Workload(Thread):
@@ -49,7 +49,7 @@ def stop(self) -> bool:
             # 'cannot join current thread'
             pass
 
-        if self.execution is not None and self.execution.status in JOB_EXEC_STATUS_ACTIVE:
+        if self.execution is not None and self.execution.status in JOB_EXEC_STATI_ACTIVE:
             update_status(self.execution, status=JOB_EXEC_STATUS_FAILED)
 
         log(f"Stopped thread {self.log_name_debug}", level=4)
@@ -61,6 +61,9 @@ def run_playbook(self):
         ansible_playbook(job=self.job, execution=self.execution)
 
     def run(self, error: bool = False) -> None:
+        if error and is_set(self.execution):
+            update_status(self.execution, status=JOB_EXEC_STATUS_RETRY)
+
         if self.once and self.started:
             self.stop()
             return
@@ -116,9 +119,12 @@ def run(self, error: bool = False) -> None:
 
         except (AnsibleConfigError, AnsibleRepositoryError, OSError) as err:
             self.config_invalid += 1
+            retry_cnt = ''
+            if not self.once:
+                retry_cnt = f" ({self.config_invalid}/{self.MAX_CONFIG_INVALID})"
+
             log(
-                msg=f"Got invalid config/environment for job {self.log_name} "
-                    f"({self.config_invalid}/{self.MAX_CONFIG_INVALID}): \"{err}\"",
+                msg=f"Got invalid config/environment for job {self.log_name}{retry_cnt}: \"{err}\"",
                 level=2,
             )
             self.run(error=True)

src/ansibleguy-webui/aw/model/base.py

@@ -5,18 +5,30 @@
     (False, 'No')
 )
 DEFAULT_NONE = {'null': True, 'default': None, 'blank': True}
-JOB_EXEC_STATUS_SUCCESS = 4
+JOB_EXEC_STATUS_WAIT = 0
+JOB_EXEC_STATUS_START = 1
+JOB_EXEC_STATUS_RUN = 2
 JOB_EXEC_STATUS_FAILED = 3
+JOB_EXEC_STATUS_SUCCESS = 4
+JOB_EXEC_STATUS_STOPPING = 5
+JOB_EXEC_STATUS_RETRY = 7
 CHOICES_JOB_EXEC_STATUS = [
-    (0, 'Waiting'),
-    (1, 'Starting'),
-    (2, 'Running'),
+    (JOB_EXEC_STATUS_WAIT, 'Waiting'),
+    (JOB_EXEC_STATUS_START, 'Starting'),
+    (JOB_EXEC_STATUS_RUN, 'Running'),
     (JOB_EXEC_STATUS_FAILED, 'Failed'),
     (JOB_EXEC_STATUS_SUCCESS, 'Finished'),
-    (5, 'Stopping'),
+    (JOB_EXEC_STATUS_STOPPING, 'Stopping'),
     (6, 'Stopped'),
+    (JOB_EXEC_STATUS_RETRY, 'Retry'),
+]
+JOB_EXEC_STATI_ACTIVE = [
+    JOB_EXEC_STATUS_WAIT,
+    JOB_EXEC_STATUS_START,
+    JOB_EXEC_STATUS_RUN,
+    JOB_EXEC_STATUS_STOPPING,
+    JOB_EXEC_STATUS_RETRY,
 ]
-JOB_EXEC_STATUS_ACTIVE = [0, 1, 2, 5]
 
 
 class BareModel(models.Model):

src/ansibleguy-webui/aw/model/repository.py

@@ -17,19 +17,19 @@ class Repository(BaseModel):
         'name', 'git_origin', 'git_credentials', 'git_branch', 'git_isolate', 'git_lfs', 'git_limit_depth',
         'git_playbook_base',
         'git_hook_pre', 'git_hook_post', 'git_hook_cleanup', 'git_override_initialize', 'git_override_update',
+        'git_timeout',
     ]
     form_fields_static = ['name', 'static_path']
     form_fields = [
         'name', 'rtype', 'static_path', 'git_origin', 'git_credentials', 'git_branch', 'git_isolate', 'git_lfs',
         'git_limit_depth', 'git_hook_pre', 'git_hook_post', 'git_hook_cleanup',
         'git_override_initialize', 'git_override_update',
-        'git_playbook_base',
+        'git_playbook_base', 'git_timeout',
     ]
     api_fields_read = form_fields.copy()
     api_fields_read.extend([
         'id', 'rtype_name', 'time_update', 'status', 'status_name', 'log_stdout', 'log_stdout_url',
         'log_stderr', 'log_stderr_url',
-
     ])
     api_fields_write = form_fields
     fields_shell_cmds = [
@@ -59,6 +59,7 @@ class Repository(BaseModel):
     git_credentials = models.ForeignKey(
         JobGlobalCredentials, on_delete=models.SET_NULL, related_name='repo_fk_cred', null=True, blank=True,
     )
+    git_timeout = models.PositiveSmallIntegerField(default=30)
 
     @property
     def rtype_name(self) -> str:

src/ansibleguy-webui/aw/static/css/aw.css

@@ -578,8 +578,8 @@ input:invalid {
     color: red;
 }
 
-.aw-job-status-finished {
-    color: green;
+.aw-job-status-retry {
+    color: orange;
 }
 
 .aw-job-status-finished {

src/ansibleguy-webui/db.py

@@ -223,7 +223,7 @@ def create_manager_groups():
 
 
 def cleanup_job_stati():
-    from aw.model.base import JOB_EXEC_STATUS_ACTIVE, JOB_EXEC_STATUS_FAILED
+    from aw.model.base import JOB_EXEC_STATI_ACTIVE, JOB_EXEC_STATUS_FAILED
     from aw.model.job import JobExecution
 
-    JobExecution.objects.filter(status__in=JOB_EXEC_STATUS_ACTIVE).update(status=JOB_EXEC_STATUS_FAILED)
+    JobExecution.objects.filter(status__in=JOB_EXEC_STATI_ACTIVE).update(status=JOB_EXEC_STATUS_FAILED)