1- #!/usr/bin/env python3
2-
3- # Copyright 2015-2020 Earth Sciences Department, BSC-CNS
4-
1+ # Copyright 2015-2025 Earth Sciences Department, BSC-CNS
2+ #
53# This file is part of Autosubmit.
6-
4+ #
75# Autosubmit is free software: you can redistribute it and/or modify
86# it under the terms of the GNU General Public License as published by
97# the Free Software Foundation, either version 3 of the License, or
108# (at your option) any later version.
11-
9+ #
1210# Autosubmit is distributed in the hope that it will be useful,
1311# but WITHOUT ANY WARRANTY; without even the implied warranty of
1412# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
1513# GNU General Public License for more details.
16-
17- from os import path
18-
14+ #
1915# You should have received a copy of the GNU General Public License
2016# along with Autosubmit. If not, see <http://www.gnu.org/licenses/>.
17+
2118import locale
2219import os
23- import psutil
2420import re
2521import shutil
2622import subprocess
2723from pathlib import Path
28- from ruamel .yaml import YAML
2924from shutil import rmtree
3025from time import time
31- from typing import List , Union
26+ from typing import List , Optional , Union
3227
33- # from autosubmit import Autosubmit
34- from autosubmit .helpers .processes import process_id
3528from autosubmitconfigparser .config .basicconfig import BasicConfig
29+ from autosubmitconfigparser .config .configcommon import AutosubmitConfig
30+
3631from log .log import Log , AutosubmitCritical
3732
3833Log .get_logger ("Autosubmit" )
4540 )''' , re .VERBOSE )
4641"""Regular expression to match Git URL."""
4742
48- class AutosubmitGit :
43+ _GIT_UNCOMMITTED_CMD = ('git' , 'status' , '--porcelain' )
44+ """Command to check if there are changes not committed go local Git repository."""
45+
46+ _GIT_UNPUSHED_CMD = ('git' , 'log' , '--branches' , '--not' , '--remotes' )
47+ """Command to check if there are changes not pushed to Git remotes."""
48+
49+
50+ def _get_uncommitted_code (git_repo : Path ) -> Optional [str ]:
51+ """Return any uncommitted changes in the given Git repository or submodules.
52+
53+ If there are no uncommitted changes, return None.
54+
55+ :param git_repo: Path to the Git repository.
56+ :return: A string containing the list of code not committed (directly git command output).
57+ Returns``None`` if there are no uncommitted changes.
4958 """
50- Class to handle experiment git repository
59+ encoding = locale .getlocale ()[1 ]
60+
61+ git_output = subprocess .check_output (_GIT_UNCOMMITTED_CMD , cwd = git_repo , encoding = encoding )
62+ for line in git_output .splitlines ():
63+ tokens = line .strip ().split (' ' )
64+ if tokens [0 ] != '' :
65+ return git_output
66+
67+ return None
68+
5169
52- :param expid: experiment identifier
53- :type expid: str
70+ def _get_code_not_pushed (git_repo : Path ) -> Optional [str ]:
71+ """Return any code not pushed to remotes in the given Git repository or submodules.
72+
73+ If there are no code not pushed changes, return None.
74+
75+ :param git_repo: Path to the Git repository.
76+ :return: A string containing the list of code not pushed (directly git command output).
77+ Returns``None`` if there are no pushed not changes.
5478 """
79+ encoding = locale .getlocale ()[1 ]
5580
56- def __init__ (self , expid ):
57- self ._expid = expid
81+ git_output = subprocess .check_output (_GIT_UNPUSHED_CMD , cwd = git_repo , encoding = encoding )
82+ for line in git_output .splitlines ():
83+ if line .strip ():
84+ return git_output
5885
59- @staticmethod
60- def clean_git (as_conf ):
61- """
62- Function to clean space on BasicConfig.LOCAL_ROOT_DIR/git directory.
86+ return None
6387
64- :param as_conf: experiment configuration
65- :type as_conf: autosubmitconfigparser.config.AutosubmitConfig
66- """
67- proj_dir = os .path .join (
68- BasicConfig .LOCAL_ROOT_DIR , as_conf .expid , BasicConfig .LOCAL_PROJ_DIR )
69- dirname_path = as_conf .get_project_dir ()
70- Log .debug ("Checking git directory status..." )
71- if path .isdir (dirname_path ):
72- if path .isdir (os .path .join (dirname_path , '.git' )):
73- try :
74- output = subprocess .check_output ("cd {0}; git diff-index HEAD --" .format (dirname_path ),
75- shell = True )
76- except subprocess .CalledProcessError as e :
77- raise AutosubmitCritical (
78- "Failed to retrieve git info ..." , 7064 , str (e ))
79- if output :
80- Log .info ("Changes not committed detected... SKIPPING!" )
81- raise AutosubmitCritical ("Commit needed!" , 7013 )
82- else :
83- output = subprocess .check_output ("cd {0}; git log --branches --not --remotes" .format (dirname_path ),
84- shell = True )
85- if output :
86- Log .info ("Changes not pushed detected... SKIPPING!" )
87- raise AutosubmitCritical (
88- "Synchronization needed!" , 7064 )
89- else :
90- if not as_conf .set_git_project_commit (as_conf ):
91- return False
92- Log .debug ("Removing directory" )
93- rmtree (proj_dir )
94- else :
95- Log .debug ("Not a git repository... SKIPPING!" )
96- else :
97- Log .debug ("Not a directory... SKIPPING!" )
98- return True
88+
89+ def check_unpushed_changes (expid : str , as_conf : AutosubmitConfig ) -> None :
90+ """Check if the Git repository is dirty for an operational experiment.
91+
92+ Raises an AutosubmitCritical error if the experiment is operational,
93+ the platform is Git, and there are unpushed changes in the local Git
94+ repository, or in any of its Git submodules.
95+
96+ :param expid: The experiment ID.
97+ :param as_conf: Autosubmit configuration object.
98+ """
99+ project_type = as_conf .get_project_type ()
100+ if expid [0 ] == 'o' and project_type == 'git' :
101+ proj_dir = Path (as_conf .get_project_dir ())
102+
103+ if uncommitted_code := _get_uncommitted_code (proj_dir ):
104+ message = ("You must commit and push your code to the remote Git repository in an "
105+ f"operational experiment before running it.\n \n { uncommitted_code } )
106+ raise AutosubmitCritical (message , 7075 )
107+
108+ if not_pushed := _get_code_not_pushed (proj_dir ):
109+ message = ("You must push your code to the remote Git repository in an "
110+ f"operational experiment before running it.\n \n { not_pushed } )
111+ raise AutosubmitCritical (message , 7075 )
112+
113+
114+ def clean_git (as_conf : AutosubmitConfig ) -> bool :
115+ """Clean the cloned Git repository inside the project directory of the experiment.
116+
117+ Skipped if the project directory location is not a valid directory.
118+
119+ Skipped if the project directory is not a valid Git repository.
120+
121+ Skipped if there are changes in the Git repository that were not committed or
122+ not pushed.
123+
124+ :param as_conf: experiment configuration
125+ :return: ``True`` if the Git project directory was successfully deleted, ``False`` otherwise.
126+ """
127+ dirname_path = Path (as_conf .get_project_dir ())
128+ Log .debug ("Checking git directory status..." )
129+
130+ if not dirname_path .is_dir ():
131+ Log .debug ("Not a directory... SKIPPING!" )
132+ return False
133+
134+ if not Path (dirname_path , '.git' ).is_dir ():
135+ Log .debug ("Not a git repository... SKIPPING!" )
136+ return False
137+
138+ if _get_uncommitted_code (dirname_path ):
139+ Log .info ("Changes not committed detected... SKIPPING!" )
140+ raise AutosubmitCritical ("Commit needed!" , 7013 )
141+
142+ if _get_code_not_pushed (dirname_path ):
143+ Log .info ("Changes not pushed detected... SKIPPING!" )
144+ raise AutosubmitCritical ("Synchronization needed!" , 7064 )
145+
146+ if not as_conf .set_git_project_commit (as_conf ):
147+ Log .info ("Failed to set Git project commit... SKIPPING!" )
148+ return False
149+
150+ proj_dir = Path (BasicConfig .LOCAL_ROOT_DIR , as_conf .expid , BasicConfig .LOCAL_PROJ_DIR )
151+ Log .debug (f"Removing project directory { str (proj_dir )} )
152+ rmtree (proj_dir )
153+
154+ return True
155+
156+
157+ class AutosubmitGit :
158+ """Class to handle experiment git repository."""
99159
100160 @staticmethod
101161 def clone_repository (as_conf , force , hpcarch ):
@@ -259,9 +319,11 @@ def clone_repository(as_conf, force, hpcarch):
259319 if os .path .exists (project_backup_path ):
260320 Log .info ("Restoring proj folder..." )
261321 shutil .move (project_backup_path , project_path )
262- raise AutosubmitCritical (f'Can not clone { git_project_branch + " " + git_project_origin } { project_path } , 7065 )
322+ raise AutosubmitCritical (
323+ f'Can not clone { git_project_branch + " " + git_project_origin } { project_path } , 7065 )
263324 if submodule_failure :
264- Log .info ("Some Submodule failures have been detected. Backup {0} will not be removed." .format (project_backup_path ))
325+ Log .info ("Some Submodule failures have been detected. Backup {0} will not be removed." .format (
326+ project_backup_path ))
265327 return False
266328
267329 if os .path .exists (project_backup_path ):
@@ -275,45 +337,3 @@ def is_git_repo(git_repo: str) -> bool:
275337 git_repo = git_repo .lower ().strip ()
276338
277339 return _GIT_URL_PATTERN .match (git_repo ) is not None
278-
279- @staticmethod
280- def check_unpushed_changes (expid : str ) -> None :
281- """
282- Raises an AutosubmitCritical error if the experiment is operational, the platform is Git, and there are unpushed changes.
283-
284- Args: expid (str): The experiment ID.
285-
286- Returns: None
287- """
288- if expid [0 ] == 'o' :
289- origin = Path (BasicConfig .expid_dir (expid ).joinpath ("conf/expdef_{}.yml" .format (expid )))
290- with open (origin , 'r' ) as f :
291- yaml = YAML (typ = 'rt' )
292- data = yaml .load (f )
293- project = data ["PROJECT" ]["PROJECT_TYPE" ]
294-
295- version_controls = ["git" ,
296- "git submodule" ]
297- arguments = [["status" , "--porcelain" ],
298- ["foreach" , "'git status --porcelain'" ]]
299- for version_control , args in zip (version_controls , arguments ):
300- if project == version_control :
301- output = subprocess .check_output ([version_control , args ]).decode (locale .getlocale ()[1 ])
302- if any (status .startswith (code ) for code in ["M" , "A" , "D" , "?" ] for status in output .splitlines ()):
303- # M: Modified, A: Added, D: Deleted, ?: Untracked
304- raise AutosubmitCritical ("Push local changes to remote repository before running" , 7075 )
305-
306- @staticmethod
307- def check_directory_in_use (expid : str ) -> bool :
308- """
309- Checks for open files and unfinished git-credential requests in a directory
310- """
311- if process_id (expid ) is not None :
312- return True
313- for proc in psutil .process_iter (['name' , 'cmdline' ]):
314- name = proc .info .get ('name' , '' )
315- cmdline = proc .info .get ('cmdline' , [])
316- if '_run.log' in name or any ('run' in arg for arg in cmdline ):
317- return True
318- return False
319-
0 commit comments