|
| 1 | +# Copyright 2015-2025 Earth Sciences Department, BSC-CNS |
| 2 | +# |
| 3 | +# This file is part of Autosubmit. |
| 4 | +# |
| 5 | +# Autosubmit is free software: you can redistribute it and/or modify |
| 6 | +# it under the terms of the GNU General Public License as published by |
| 7 | +# the Free Software Foundation, either version 3 of the License, or |
| 8 | +# (at your option) any later version. |
| 9 | +# |
| 10 | +# Autosubmit is distributed in the hope that it will be useful, |
| 11 | +# but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 12 | +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 13 | +# GNU General Public License for more details. |
| 14 | +# |
| 15 | +# You should have received a copy of the GNU General Public License |
| 16 | +# along with Autosubmit. If not, see <http://www.gnu.org/licenses/>. |
| 17 | + |
| 18 | + |
| 19 | +from pathlib import Path |
| 20 | + |
| 21 | +import pytest |
| 22 | + |
| 23 | +from autosubmit.job.job import Job |
| 24 | +from autosubmit.job.job_common import Status |
| 25 | +from autosubmit.job.job_packages import JobPackageSimple |
| 26 | +from autosubmit.log.log import AutosubmitCritical, AutosubmitError |
| 27 | +from autosubmit.platforms.pbsplatform import PBSPlatform |
| 28 | + |
| 29 | +"""Unit tests for the PBS platform.""" |
| 30 | + |
| 31 | + |
| 32 | +@pytest.fixture |
| 33 | +def platform(autosubmit_config): |
| 34 | + expid = 'a000' |
| 35 | + as_conf = autosubmit_config(expid, experiment_data={}) |
| 36 | + exp_path = Path(as_conf.basic_config.LOCAL_ROOT_DIR, expid) |
| 37 | + aslogs_dir = exp_path / as_conf.basic_config.LOCAL_TMP_DIR / as_conf.basic_config.LOCAL_ASLOG_DIR |
| 38 | + submit_platform_script = aslogs_dir / 'submit_local.sh' |
| 39 | + Path(submit_platform_script).touch() |
| 40 | + return PBSPlatform(expid='a000', name='local', config=as_conf.experiment_data) |
| 41 | + |
| 42 | + |
| 43 | +def test_properties(platform): |
| 44 | + props = { |
| 45 | + 'name': 'foo', |
| 46 | + 'host': 'localhost1', |
| 47 | + 'user': 'sam', |
| 48 | + 'project': 'proj1', |
| 49 | + 'budget': 100, |
| 50 | + 'reservation': 1, |
| 51 | + 'exclusivity': True, |
| 52 | + 'hyperthreading': True, |
| 53 | + 'type': 'SuperPBS', |
| 54 | + 'scratch': '/scratch/1', |
| 55 | + 'project_dir': '/proj1', |
| 56 | + 'root_dir': '/root_1', |
| 57 | + 'partition': 'inter', |
| 58 | + 'queue': 'prio1' |
| 59 | + } |
| 60 | + for prop, value in props.items(): |
| 61 | + setattr(platform, prop, value) |
| 62 | + for prop, value in props.items(): |
| 63 | + assert value == getattr(platform, prop) |
| 64 | + |
| 65 | + |
| 66 | +def test_pbs_platform_submit_script_raises_autosubmit_critical_with_trace(mocker, platform): |
| 67 | + package = mocker.MagicMock() |
| 68 | + package.jobs.return_value = [] |
| 69 | + valid_packages_to_submit = [ |
| 70 | + package |
| 71 | + ] |
| 72 | + |
| 73 | + ae = AutosubmitError(message='violates resource limits', code=123, trace='ERR!') |
| 74 | + platform.submit_script = mocker.MagicMock(side_effect=ae) |
| 75 | + |
| 76 | + # AS will handle the AutosubmitError above, but then raise an AutosubmitCritical. |
| 77 | + # This new error won't contain all the info from the upstream error. |
| 78 | + with pytest.raises(AutosubmitCritical) as cm: |
| 79 | + platform.process_batch_ready_jobs( |
| 80 | + valid_packages_to_submit=valid_packages_to_submit, |
| 81 | + failed_packages=[] |
| 82 | + ) |
| 83 | + |
| 84 | + # AS will handle the error and then later will raise another error message. |
| 85 | + # But the AutosubmitError object we created will have been correctly used |
| 86 | + # without raising any exceptions (such as AttributeError). |
| 87 | + assert cm.value.message != ae.message |
| 88 | + |
| 89 | + |
| 90 | +@pytest.fixture |
| 91 | +def as_conf(autosubmit_config, tmpdir): |
| 92 | + exp_data = { |
| 93 | + "PLATFORMS": { |
| 94 | + "pytest-pbs": { |
| 95 | + "type": "pbs", |
| 96 | + "host": "localhost", |
| 97 | + "user": "user", |
| 98 | + "project": "project", |
| 99 | + "scratch_dir": "/scratch", |
| 100 | + "QUEUE": "queue", |
| 101 | + "ADD_PROJECT_TO_HOST": False, |
| 102 | + "MAX_WALLCLOCK": "00:01", |
| 103 | + "TEMP_DIR": "", |
| 104 | + "MAX_PROCESSORS": 99999, |
| 105 | + }, |
| 106 | + }, |
| 107 | + "LOCAL_ROOT_DIR": str(tmpdir), |
| 108 | + "LOCAL_TMP_DIR": str(tmpdir), |
| 109 | + "LOCAL_PROJ_DIR": str(tmpdir), |
| 110 | + "LOCAL_ASLOG_DIR": str(tmpdir), |
| 111 | + } |
| 112 | + as_conf = autosubmit_config("dummy-expid", exp_data) |
| 113 | + return as_conf |
| 114 | + |
| 115 | + |
| 116 | +@pytest.fixture |
| 117 | +def pbs_platform(as_conf): |
| 118 | + platform = PBSPlatform(expid="dummy-expid", name='pytest-pbs', config=as_conf.experiment_data) |
| 119 | + return platform |
| 120 | + |
| 121 | + |
| 122 | +@pytest.fixture |
| 123 | +def create_packages(as_conf, pbs_platform): |
| 124 | + simple_jobs_1 = [Job("dummy-1", 1, Status.SUBMITTED, 0)] |
| 125 | + simple_jobs_2 = [Job("dummy-1", 1, Status.SUBMITTED, 0), |
| 126 | + Job("dummy-2", 2, Status.SUBMITTED, 0), |
| 127 | + Job("dummy-3", 3, Status.SUBMITTED, 0)] |
| 128 | + simple_jobs_3 = [Job("dummy-1", 1, Status.SUBMITTED, 0), |
| 129 | + Job("dummy-2", 2, Status.SUBMITTED, 0), |
| 130 | + Job("dummy-3", 3, Status.SUBMITTED, 0)] |
| 131 | + for job in simple_jobs_1 + simple_jobs_2 + simple_jobs_3: |
| 132 | + job._platform = pbs_platform |
| 133 | + job._platform.name = pbs_platform.name |
| 134 | + job.platform_name = pbs_platform.name |
| 135 | + job.processors = 2 |
| 136 | + job.section = "dummysection" |
| 137 | + job._init_runtime_parameters() |
| 138 | + job.wallclock = "00:01" |
| 139 | + packages = [ |
| 140 | + JobPackageSimple(simple_jobs_1), |
| 141 | + JobPackageSimple(simple_jobs_2), |
| 142 | + JobPackageSimple(simple_jobs_3), |
| 143 | + ] |
| 144 | + return packages |
| 145 | + |
| 146 | + |
| 147 | +def test_process_batch_ready_jobs_valid_packages_to_submit(mocker, pbs_platform, as_conf, create_packages): |
| 148 | + valid_packages_to_submit = create_packages |
| 149 | + failed_packages = [] |
| 150 | + pbs_platform.get_jobs_id_by_job_name = mocker.MagicMock() |
| 151 | + pbs_platform.send_command = mocker.MagicMock() |
| 152 | + pbs_platform.submit_script = mocker.MagicMock() |
| 153 | + jobs_id = [1, [1, 2, 3], [1, 2, 3]] |
| 154 | + pbs_platform.get_jobs_id_by_job_name.return_value = jobs_id |
| 155 | + pbs_platform.submit_script.return_value = jobs_id |
| 156 | + pbs_platform.process_batch_ready_jobs(valid_packages_to_submit, failed_packages) |
| 157 | + for i, package in enumerate(valid_packages_to_submit): |
| 158 | + for job in package.jobs: |
| 159 | + assert job.hold is False |
| 160 | + assert job.id == str(jobs_id[i]) |
| 161 | + assert job.status == Status.SUBMITTED |
| 162 | + assert job.wrapper_name is None |
| 163 | + assert failed_packages == [] |
| 164 | + |
| 165 | + |
| 166 | +def test_submit_job(mocker, pbs_platform): |
| 167 | + pbs_platform.get_submit_cmd = mocker.MagicMock(returns="dummy") |
| 168 | + pbs_platform.send_command = mocker.MagicMock(returns="dummy") |
| 169 | + pbs_platform._ssh_output = "10000" |
| 170 | + job = Job("dummy", 10000, Status.SUBMITTED, 0) |
| 171 | + job._platform = pbs_platform |
| 172 | + job.platform_name = pbs_platform.name |
| 173 | + jobs_id = pbs_platform.submit_job(job, "dummy") |
| 174 | + assert not jobs_id |
| 175 | + job.x11 = True |
| 176 | + jobs_id = pbs_platform.submit_job(job, "dummy") |
| 177 | + assert jobs_id == 10000 |
| 178 | + job.workflow_commit = "dummy" |
| 179 | + jobs_id = pbs_platform.submit_job(job, "dummy") |
| 180 | + assert jobs_id == 10000 |
| 181 | + pbs_platform._ssh_output = "10000\n" |
| 182 | + jobs_id = pbs_platform.submit_job(job, "dummy") |
| 183 | + assert jobs_id == 10000 |
0 commit comments