Skip to content

Commit ef99016

Browse files
VindeeRVindeeR
committed
f
1 parent 33ac0e5 commit ef99016

19 files changed

+55
-338
lines changed

‎autosubmit/autosubmit.py‎

Lines changed: 38 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -39,12 +39,11 @@
3939
from importlib.resources import files as read_files
4040
from pathlib import Path
4141
from time import sleep
42-
from typing import Any, Optional, Union, TYPE_CHECKING
42+
from typing import Any, Optional, Union
4343

4444
from bscearth.utils.date import date2str
4545
from portalocker import Lock
4646
from portalocker.exceptions import BaseLockException
47-
from pycparser.c_ast import Return
4847
from pyparsing import nestedExpr
4948
from ruamel.yaml import YAML
5049

@@ -81,7 +80,6 @@
8180
from autosubmit.platforms.paramiko_submitter import ParamikoSubmitter
8281
from autosubmit.platforms.platform import Platform
8382

84-
8583
dialog = None
8684

8785
"""Main module for autosubmit. Only contains an interface class to all functionality implemented on autosubmit."""
@@ -492,14 +490,15 @@ def parse_args() -> tuple[int, Optional[argparse.Namespace]]:
492490
help="Select the database backend to use. Default is sqlite.",
493491
)
494492
subparser.add_argument(
495-
"--database-conn-url", default=None, help="Database connection URL string. Required for postgres backend."
493+
"--database-conn-url", default=None,
494+
help="Database connection URL string. Required for postgres backend."
496495
)
497496
subparser.add_argument(
498497
"-db",
499498
"--databasepath",
500499
default=None,
501500
help="path to SQLite database. If not supplied, "
502-
"it will prompt for it. Required for SQLite backend.",
501+
"it will prompt for it. Required for SQLite backend.",
503502
)
504503
subparser.add_argument(
505504
'-dbf', '--databasefilename', default=None, help='database filename')
@@ -888,12 +887,12 @@ def _init_logs(args, console_level='INFO', log_level='DEBUG', expid='None'):
888887
if not permissions:
889888
raise AutosubmitCritical(
890889
f'Experiments database {BasicConfig.DB_PATH} not readable. Please check permissions.'
891-
, 7007)
890+
, 7007)
892891
permissions = os.access(BasicConfig.DB_PATH, os.W_OK) # Check for write access
893892
if not permissions:
894893
raise AutosubmitCritical(
895894
f'Experiments database {BasicConfig.DB_PATH} not writable. Please check permissions.'
896-
, 7007)
895+
, 7007)
897896

898897
expid_less = ["expid", "describe", "testcase", "install", "-v",
899898
"readme", "changelog", "configure", "unarchive",
@@ -1243,7 +1242,6 @@ def _perform_deletion(experiment_path: Path, structure_db_path: Path, job_data_d
12431242
except BaseException as e:
12441243
error_message += f"Cannot delete directory: {e}\n"
12451244

1246-
12471245
if is_sqlite:
12481246
Log.info("Removing Structure db...")
12491247
try:
@@ -2197,7 +2195,7 @@ def prepare_run(
21972195
except IOError as e:
21982196
raise AutosubmitError("job_packages not found", 6016, str(e))
21992197
# Check if the user wants to continue using wrappers and loads the appropriate info.
2200-
if as_conf.experiment_data.get("WRAPPERS",None) is not None:
2198+
if as_conf.experiment_data.get("WRAPPERS", None) is not None:
22012199
if BasicConfig.DATABASE_BACKEND == 'sqlite':
22022200
os.chmod(os.path.join(BasicConfig.LOCAL_ROOT_DIR,
22032201
expid, "pkl", "job_packages_" + expid + ".db"), 0o644)
@@ -2637,21 +2635,24 @@ def restore_platforms(platforms_to_test, mail_notify=False, as_conf=None, expid=
26372635
if platform_to_test.check_remote_permissions():
26382636
Log.result(f"[{platform_to_test.name}] Correct user privileges for host {platform_to_test.host}")
26392637
else:
2640-
platform_issues += (f"\n[{platform_to_test.name}] has configuration issues.\n Check that the connection is"
2641-
f" passwd-less.(ssh {platform_to_test.user}@{platform_to_test.host})\n Check the parameters that"
2642-
f" build the root_path are correct:{{scratch_dir/project/user}} ="
2643-
f" {{{platform_to_test.scratch}/{platform_to_test.project}/{platform_to_test.user}}}")
2638+
platform_issues += (
2639+
f"\n[{platform_to_test.name}] has configuration issues.\n Check that the connection is"
2640+
f" passwd-less.(ssh {platform_to_test.user}@{platform_to_test.host})\n Check the parameters that"
2641+
f" build the root_path are correct:{{scratch_dir/project/user}} ="
2642+
f" {{{platform_to_test.scratch}/{platform_to_test.project}/{platform_to_test.user}}}")
26442643
issues += platform_issues
26452644
if platform_issues == "":
26462645

2647-
Log.printlog(f"[{platform_to_test.name}] Connection successful to host {platform_to_test.host}", Log.RESULT)
2646+
Log.printlog(f"[{platform_to_test.name}] Connection successful to host {platform_to_test.host}",
2647+
Log.RESULT)
26482648
else:
26492649
if platform_to_test.connected:
26502650
platform_to_test.connected = False
26512651
Log.printlog(f"[{platform_to_test.name}] Connection successful to host {platform_to_test.host}, "
26522652
f"however there are issues with %HPCROOT%", Log.WARNING)
26532653
else:
2654-
Log.printlog(f"[{platform_to_test.name}] Connection failed to host {platform_to_test.host}", Log.WARNING)
2654+
Log.printlog(f"[{platform_to_test.name}] Connection failed to host {platform_to_test.host}",
2655+
Log.WARNING)
26552656
if issues != "":
26562657
if ssh_config_issues.find(private_key_error[:-2]) != -1:
26572658
raise AutosubmitCritical("Private key is encrypted, Autosubmit does not run in "
@@ -2855,7 +2856,8 @@ def monitor(expid: str, file_format: str, lst: str, filter_chunks: str, filter_s
28552856
packages_persistence = JobPackagePersistence(expid)
28562857
# Permissions
28572858
if BasicConfig.DATABASE_BACKEND == 'sqlite':
2858-
os.chmod(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid, "pkl", "job_packages_" + expid + ".db"), 0o644)
2859+
os.chmod(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid, "pkl", "job_packages_" + expid + ".db"),
2860+
0o644)
28592861
# Database modification
28602862
packages_persistence.reset_table(True)
28612863
# Load another job_list to go through that goes through the jobs, but we want to monitor the other one
@@ -3528,18 +3530,18 @@ def describe(input_experiment_list="*", get_from_user=""):
35283530

35293531
@staticmethod
35303532
def configure(
3531-
advanced,
3532-
database_path,
3533-
database_filename,
3534-
local_root_path,
3535-
platforms_conf_path,
3536-
jobs_conf_path,
3537-
smtp_hostname,
3538-
mail_from,
3539-
machine: bool,
3540-
local: bool,
3541-
database_backend: str = "sqlite",
3542-
database_conn_url: str = "",
3533+
advanced,
3534+
database_path,
3535+
database_filename,
3536+
local_root_path,
3537+
platforms_conf_path,
3538+
jobs_conf_path,
3539+
smtp_hostname,
3540+
mail_from,
3541+
machine: bool,
3542+
local: bool,
3543+
database_backend: str = "sqlite",
3544+
database_conn_url: str = "",
35433545
):
35443546
"""
35453547
Configure several paths for autosubmit: database, local root and others. Can be configured at system,
@@ -4137,7 +4139,8 @@ def pkl_fix(expid: str, force: bool = False) -> int:
41374139
:rtype:
41384140
"""
41394141
if BasicConfig.DATABASE_BACKEND != 'sqlite':
4140-
raise AutosubmitCritical("This operation is only available when Autosubmit database backend is set to sqlite", 7000)
4142+
raise AutosubmitCritical(
4143+
"This operation is only available when Autosubmit database backend is set to sqlite", 7000)
41414144
exp_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid)
41424145
tmp_path = os.path.join(exp_path, BasicConfig.LOCAL_TMP_DIR)
41434146
pkl_folder_path = os.path.join(exp_path, "pkl")
@@ -4608,7 +4611,7 @@ def create(expid: str, noplot: bool, hide: bool, output='pdf', group_by: Optiona
46084611
as_conf.check_conf_files(running_time=False, force_load=True, no_log=False)
46094612
if len(as_conf.experiment_data.get("JOBS",
46104613
{})) == 0 and "CUSTOM_CONFIG" in as_conf.experiment_data.get(
4611-
"DEFAULT", {}):
4614+
"DEFAULT", {}):
46124615
raise AutosubmitCritical(
46134616
f'Job list is empty\nCheck if there are YML files in {as_conf.experiment_data.get("DEFAULT", "").get("CUSTOM_CONFIG", "")}',
46144617
code=7015)
@@ -5221,7 +5224,7 @@ def _apply_ftc(job_list: JobList, filter_type_chunk_split: str) -> list[Job]:
52215224
final_splits.append(str(split))
52225225
splits = final_splits
52235226
jobs_filtered = [j for j in working_list if j.section == section and (
5224-
j.split is None or splits[0] == "ANY" or str(j.split) in splits)]
5227+
j.split is None or splits[0] == "ANY" or str(j.split) in splits)]
52255228
# Go through start dates
52265229
for starting_date in deserialized_json['sds']:
52275230
date = starting_date['sd']
@@ -5497,7 +5500,7 @@ def set_status(expid: str, noplot: bool, save: bool, final: str, filter_list: st
54975500
packages_persistence = JobPackagePersistence(expid)
54985501
if BasicConfig.DATABASE_BACKEND == 'sqlite':
54995502
os.chmod(os.path.join(BasicConfig.LOCAL_ROOT_DIR,
5500-
expid, "pkl", "job_packages_" + expid + ".db"), 0o775)
5503+
expid, "pkl", "job_packages_" + expid + ".db"), 0o775)
55015504
packages_persistence.reset_table(True)
55025505
job_list_wr = Autosubmit.load_job_list(expid, as_conf, monitor=True, new=False)
55035506

@@ -5720,7 +5723,9 @@ def _get_job_list_persistence(expid, as_conf) -> JobListPersistence:
57205723
raise AutosubmitCritical('Storage type not known', 7014)
57215724

57225725
@staticmethod
5723-
def testcase(description, chunks=None, member=None, start_date=None, hpc=None, copy_id=None, minimal_configuration=False, git_repo=None, git_branch=None, git_as_conf=None, use_local_minimal=False):
5726+
def testcase(description, chunks=None, member=None, start_date=None, hpc=None, copy_id=None,
5727+
minimal_configuration=False, git_repo=None, git_branch=None, git_as_conf=None,
5728+
use_local_minimal=False):
57245729
"""
57255730
Method to conduct a test for a given experiment. It creates a new experiment for a given experiment with a
57265731
given number of chunks with a random start date and a random member to be run on a random HPC.

‎autosubmit/job/job.py‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,10 @@
3636
from autosubmit.config.configcommon import AutosubmitConfig
3737
from autosubmit.helpers.parameters import autosubmit_parameter, autosubmit_parameters
3838
from autosubmit.history.experiment_history import ExperimentHistory
39-
from autosubmit.job.job_common import Status, Type, increase_wallclock_by_chunk
39+
from autosubmit.job.job_common import Status, increase_wallclock_by_chunk
4040
from autosubmit.job.job_utils import get_job_package_code, get_split_size_unit, get_split_size
4141
from autosubmit.job.metrics_processor import UserMetricProcessor
42-
from autosubmit.job.template import get_template_snippet, Language
42+
from autosubmit.job.template import get_template_snippet, Language, TemplateSnippet
4343
from autosubmit.log.log import Log, AutosubmitCritical
4444
from autosubmit.platforms.paramiko_platform import ParamikoPlatform
4545
from autosubmit.platforms.paramiko_submitter import ParamikoSubmitter
@@ -1425,10 +1425,7 @@ def update_status(self, as_conf: AutosubmitConfig, failed_file: bool = False) ->
14251425
new_status = self.new_status
14261426
if new_status == Status.COMPLETED:
14271427
Log.debug(f"{self.name} job seems to have completed: checking...")
1428-
if not self._platform.get_completed_files(self.name, wrapper_failed=self.packed):
1429-
# FIXME: log_name not used?
1430-
log_name = os.path.join(self._tmp_path, self.name + '_COMPLETED')
1431-
1428+
self._platform.get_completed_files(self.name, wrapper_failed=self.packed)
14321429
self.check_completion()
14331430
else:
14341431
self.status = new_status

‎autosubmit/job/job_common.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ def parse_output_number(string_number):
117117
number = string_number
118118
try:
119119
number = float(number) * multiplier
120-
except Exception as exp:
120+
except Exception:
121121
number = 0.0
122122
pass
123123
return number

‎autosubmit/job/job_packages.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -391,7 +391,7 @@ def _do_submission(self, job_scripts: dict[str, str] = None, hold: bool = False)
391391
return
392392

393393
for i in range(0, len(self.jobs)): # platforms without a submit.cmd
394-
LLog.info(f"{self.jobs[i].name} submitted")
394+
Log.info(f"{self.jobs[i].name} submitted")
395395
self.jobs[i].id = str(package_id) + f'[{i}]'
396396
self.jobs[i].status = Status.SUBMITTED
397397
# Identify to which wrapper this job belongs once it is in the recovery queue

‎autosubmit/platforms/headers/slurm_header.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@
1919

2020
import textwrap
2121

22+
from autosubmit.job.job import Job
23+
24+
2225
class SlurmHeader(object):
2326
"""Class to handle the SLURM headers of a job"""
2427

‎autosubmit/platforms/paramiko_platform.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -352,7 +352,7 @@ def connect(self, as_conf: 'AutosubmitConfig', reconnect: bool = False, log_reco
352352
self._ssh.connect(self._host_config['hostname'], port, username=self.user,
353353
key_filename=self._host_config_id, sock=self._proxy, timeout=60,
354354
banner_timeout=60)
355-
except Exception:
355+
except Exception as e:
356356
Log.warning(f'Failed to SSH connect to {self._host_config["hostname"]}: {e}')
357357
Log.warning('Will try disabling the rsa-sha2-256 and rsa-sha2-512 SSH '
358358
'public key algorithms...')
@@ -385,7 +385,7 @@ def connect(self, as_conf: 'AutosubmitConfig', reconnect: bool = False, log_reco
385385
self.transport.start_client()
386386
try:
387387
self.transport.auth_interactive(self.user, self.interactive_auth_handler)
388-
except Exception:
388+
except Exception as e:
389389
Log.printlog(f"2FA authentication failed: {str(e)}", 7000)
390390
raise
391391
if self.transport.is_authenticated():

‎autosubmit/platforms/paramiko_submitter.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525

2626
import os
2727
from collections import defaultdict
28-
from typing import Optional, Union, TYPE_CHECKING
28+
from typing import Optional, Union
2929

3030
from autosubmit.config.basicconfig import BasicConfig
3131
from autosubmit.config.configcommon import AutosubmitConfig
@@ -189,7 +189,7 @@ def load_platforms(self, as_conf: 'AutosubmitConfig',
189189
remote_platform.update_cmds()
190190

191191
self.platforms[section] = remote_platform
192-
except Exception:
192+
except Exception as e:
193193
raise_message = (f"Error in the definition of PLATFORM in YAML: "
194194
f"SCRATCH_DIR, PROJECT, USER, EXPID must be defined for platform {section}: {str(e)}")
195195
# Executes update_cmds() from corresponding Platform Object

‎autosubmit/platforms/wrappers/wrapper_factory.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -229,7 +229,7 @@ def queue_directive(self, queue):
229229
def partition_directive(self, partition):
230230
return f'#SBATCH --partition={partition}'
231231
def exclusive_directive(self, exclusive):
232-
return f'#SBATCH --exclusive'
232+
return '#SBATCH --exclusive'
233233
def tasks_directive(self, tasks):
234234
return f'#SBATCH --ntasks-per-node={tasks}'
235235
def nodes_directive(self, nodes):

‎test/integration/database/test_db_manager.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
from autosubmit.database.db_common import get_connection_url
2626
from autosubmit.database.db_manager import DbManager
2727
from autosubmit.database.tables import DBVersionTable
28-
from test.integration.conftest import as_db
2928

3029
if TYPE_CHECKING:
3130
# noinspection PyProtectedMember

‎test/integration/platforms/test_slurmplatform.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
"""
2828

2929
import pytest
30-
from typing import TYPE_CHECKING
3130

3231
from autosubmit.config.configcommon import AutosubmitConfig
3332
from autosubmit.config.yamlparser import YAMLParserFactory

0 commit comments

Comments
 (0)