Skip to content

Commit 86d3cc5

Browse files
ehoganehogan
authored
#270: Update CDDS version
* #223: Update request, use configparser to write request file, update CDDS version * #223: Ignore warnings from pytest * #223: Add root directories to request * #223: Add variable file to request * #223: Update request file name * #223: Restore overeagerly removed commands * #223: Update ESMValTool version and CMIP6 root path * #223: Move cdds_convert arguments to request configuration file * #223: Update docstring
1 parent d4bc616 commit 86d3cc5

7 files changed

Lines changed: 119 additions & 85 deletions

File tree

‎CMEW/app/configure_standardise/bin/configure_standardise.sh‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,10 @@ set -eux
77

88
echo "Running configure_standardise"
99

10+
# Create request configuration file and variables file.
1011
cmew-process-env create_request_file.py
11-
1212
cmew-process-env create_variables_file.py
1313

14-
# Running setup commands for CDDS
15-
16-
cmew-standardise-env create_cdds_directory_structure "${REQUEST_PATH}" -c "${ROOT_PROC_DIR}" -t "${ROOT_DATA_DIR}"
17-
18-
cmew-standardise-env prepare_generate_variable_list "${REQUEST_PATH}" -c "${ROOT_PROC_DIR}" -t "${ROOT_DATA_DIR}" --use_proc_dir -r "${VARIABLES_PATH}"
14+
# Create CDDS directory structure and variables list.
15+
cmew-standardise-env create_cdds_directory_structure "${REQUEST_PATH}"
16+
cmew-standardise-env prepare_generate_variable_list "${REQUEST_PATH}"

‎CMEW/app/configure_standardise/bin/create_request_file.py‎

Lines changed: 45 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -2,84 +2,90 @@
22
# (C) Crown Copyright 2024-2025, Met Office.
33
# The LICENSE.md file contains full licensing details.
44
"""
5-
Generates the request.json file from the ESMValTool recipe.
5+
Generates the request configuration file from the ESMValTool recipe.
66
"""
7+
import configparser
78
import os
89
from pathlib import Path
9-
import json
1010

1111

1212
def create_request():
1313
"""Retrieve CDDS request information from Rose suite configuration.
1414
1515
Returns
1616
-------
17-
dict
18-
CDDS request information to be written to JSON file.
17+
configparser.ConfigParser()
18+
CDDS request configuration.
1919
"""
20-
streams = ["apm"]
21-
start_datetime = f"{os.environ['START_YEAR']}-01-01T00:00:00"
2220
end_year = int(os.environ["START_YEAR"]) + int(
2321
os.environ["NUMBER_OF_YEARS"]
2422
)
25-
end_datetime = f"{end_year}-01-01T00:00:00"
26-
run_bounds = f"{start_datetime} {end_datetime}"
27-
streams_run_bounds = {
28-
f"run_bounds_for_stream_{stream}": run_bounds for stream in streams
29-
}
30-
request = {
31-
"atmos_timestep": "1200",
23+
request = configparser.ConfigParser()
24+
request["metadata"] = {
25+
"base_date": "1850-01-01T00:00:00",
3226
"branch_method": "no parent",
3327
"calendar": os.environ["CALENDAR"],
34-
"child_base_date": "1850-01-01T00:00:00",
35-
"config_version": "1.0.1",
3628
"experiment_id": "amip",
37-
"external_plugin": "",
38-
"external_plugin_location": "",
39-
"global_attributes": {},
4029
"institution_id": os.environ["INSTITUTION_ID"],
4130
"license": "GCModelDev model data is licensed under the Open Government License v3 (https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/)", # noqa: E501
42-
"mass_data_class": "crum",
4331
"mip": "ESMVal",
4432
"mip_era": "GCModelDev",
45-
"mip_table_dir": os.path.expanduser(
46-
"~cdds/etc/mip_tables/GCModelDev/0.0.9"
47-
),
4833
"model_id": os.environ["MODEL_ID"],
4934
"model_type": "AGCM AER",
50-
"package": "round-1",
51-
"request_id": "CMEW",
52-
"run_bounds": run_bounds,
5335
"sub_experiment_id": "none",
54-
"suite_branch": "trunk",
55-
"suite_id": os.environ["SUITE_ID"],
56-
"suite_revision": "not used except with data request",
5736
"variant_label": "r1i1p1f1",
5837
}
59-
# Combine request dict and streams_run_bounds dict (in-place union).
60-
request |= streams_run_bounds
38+
request["common"] = {
39+
"external_plugin": "",
40+
"external_plugin_location": "",
41+
"mip_table_dir": os.path.expanduser(
42+
"~cdds/etc/mip_tables/GCModelDev/0.0.9"
43+
),
44+
"mode": "relaxed",
45+
"package": "round-1",
46+
"root_proc_dir": os.environ["ROOT_PROC_DIR"],
47+
"root_data_dir": os.environ["ROOT_DATA_DIR"],
48+
"workflow_basename": "CMEW",
49+
}
50+
request["data"] = {
51+
"end_date": f"{end_year}-01-01T00:00:00",
52+
"mass_data_class": "crum",
53+
"model_workflow_branch": "trunk",
54+
"model_workflow_id": os.environ["SUITE_ID"],
55+
"model_workflow_revision": "not used except with data request",
56+
"start_date": f"{os.environ['START_YEAR']}-01-01T00:00:00",
57+
"streams": "apm",
58+
"variable_list_file": os.environ["VARIABLES_PATH"],
59+
}
60+
request["misc"] = {
61+
"atmos_timestep": "1200",
62+
}
63+
request["conversion"] = {
64+
"mip_convert_plugin": "UKESM1",
65+
"skip_archive": "True",
66+
"cylc_args": "--no-detach -v",
67+
}
6168
return request
6269

6370

6471
def write_request(request, target_path):
65-
"""Write request dictionary to a JSON file at ``target_path``.
72+
"""Write the request configuration to a file at ``target_path``.
6673
6774
Parameters
6875
----------
69-
request : dict
70-
Dictionary containing the request information.
76+
request : configparser.ConfigParser()
77+
The request configuration.
7178
7279
target_path: Path
73-
Location to write the request file.
80+
The full path to the file
81+
where the request configuration will be written.
7482
"""
75-
with open(target_path, mode="w") as file:
76-
json.dump(request, file, separators=(",\n", ": "))
83+
with open(target_path, mode="w") as file_handle:
84+
request.write(file_handle)
7785

7886

7987
def main():
80-
target_path = (
81-
Path(os.environ["CYLC_WORKFLOW_SHARE_DIR"]) / "etc" / "request.json"
82-
)
88+
target_path = Path(os.environ["REQUEST_PATH"])
8389
request = create_request()
8490
write_request(request, target_path)
8591

‎CMEW/app/configure_standardise/bin/test_create_request_file.py‎

Lines changed: 54 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -6,42 +6,65 @@
66

77

88
def test_create_request(monkeypatch):
9+
# In the order defined in 'create_request_file.py'.
10+
monkeypatch.setenv("START_YEAR", "1993")
11+
monkeypatch.setenv("NUMBER_OF_YEARS", "1")
912
monkeypatch.setenv("CALENDAR", "360_day")
1013
monkeypatch.setenv("INSTITUTION_ID", "MOHC")
1114
monkeypatch.setenv("MODEL_ID", "UKESM1-0-LL")
12-
monkeypatch.setenv("START_YEAR", "1993")
13-
monkeypatch.setenv("NUMBER_OF_YEARS", "1")
15+
monkeypatch.setenv("ROOT_PROC_DIR", "/path/to/proc/dir/")
16+
monkeypatch.setenv("ROOT_DATA_DIR", "/path/to/data/dir/")
1417
monkeypatch.setenv("SUITE_ID", "u-az513")
18+
monkeypatch.setenv("VARIABLES_PATH", "/path/to/variables.txt")
1519

16-
actual = create_request()
20+
config = create_request()
21+
actual = {
22+
section: dict(config.items(section)) for section in config.sections()
23+
}
1724
expected = {
18-
"atmos_timestep": "1200",
19-
"branch_method": "no parent",
20-
"calendar": "360_day",
21-
"child_base_date": "1850-01-01T00:00:00",
22-
"config_version": "1.0.1",
23-
"experiment_id": "amip",
24-
"external_plugin": "",
25-
"external_plugin_location": "",
26-
"global_attributes": {},
27-
"institution_id": "MOHC",
28-
"license": "GCModelDev model data is licensed under the Open Government License v3 (https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/)", # noqa: E501
29-
"mass_data_class": "crum",
30-
"mip": "ESMVal",
31-
"mip_era": "GCModelDev",
32-
"mip_table_dir": os.path.expanduser(
33-
"~cdds/etc/mip_tables/GCModelDev/0.0.9"
34-
),
35-
"model_id": "UKESM1-0-LL",
36-
"model_type": "AGCM AER",
37-
"package": "round-1",
38-
"request_id": "CMEW",
39-
"run_bounds": "1993-01-01T00:00:00 1994-01-01T00:00:00",
40-
"run_bounds_for_stream_apm": "1993-01-01T00:00:00 1994-01-01T00:00:00",
41-
"sub_experiment_id": "none",
42-
"suite_branch": "trunk",
43-
"suite_id": "u-az513",
44-
"suite_revision": "not used except with data request",
45-
"variant_label": "r1i1p1f1",
25+
"metadata": {
26+
"branch_method": "no parent",
27+
"calendar": "360_day",
28+
"base_date": "1850-01-01T00:00:00",
29+
"experiment_id": "amip",
30+
"institution_id": "MOHC",
31+
"license": "GCModelDev model data is licensed under the Open Government License v3 (https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/)", # noqa: E501
32+
"mip": "ESMVal",
33+
"mip_era": "GCModelDev",
34+
"model_id": "UKESM1-0-LL",
35+
"model_type": "AGCM AER",
36+
"sub_experiment_id": "none",
37+
"variant_label": "r1i1p1f1",
38+
},
39+
"common": {
40+
"external_plugin": "",
41+
"external_plugin_location": "",
42+
"mip_table_dir": os.path.expanduser(
43+
"~cdds/etc/mip_tables/GCModelDev/0.0.9"
44+
),
45+
"mode": "relaxed",
46+
"package": "round-1",
47+
"root_proc_dir": "/path/to/proc/dir/",
48+
"root_data_dir": "/path/to/data/dir/",
49+
"workflow_basename": "CMEW",
50+
},
51+
"data": {
52+
"end_date": "1994-01-01T00:00:00",
53+
"mass_data_class": "crum",
54+
"model_workflow_branch": "trunk",
55+
"model_workflow_id": "u-az513",
56+
"model_workflow_revision": "not used except with data request",
57+
"start_date": "1993-01-01T00:00:00",
58+
"streams": "apm",
59+
"variable_list_file": "/path/to/variables.txt",
60+
},
61+
"misc": {
62+
"atmos_timestep": "1200",
63+
},
64+
"conversion": {
65+
"mip_convert_plugin": "UKESM1",
66+
"skip_archive": "True",
67+
"cylc_args": "--no-detach -v",
68+
},
4669
}
4770
assert actual == expected

‎CMEW/app/standardise_model_data/rose-app.conf‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,5 @@
22
# The LICENSE.md file contains full licensing details.
33

44
[command]
5-
default=cmew-standardise-env cdds_convert ${REQUEST_PATH} \
6-
=--skip_transfer --relaxed_cmor \
7-
=-c ${ROOT_PROC_DIR} \
8-
=-t ${ROOT_DATA_DIR} \
9-
=--cylc_args="--no-detach"
5+
default=cmew-standardise-env cdds_convert ${REQUEST_PATH}
106
=cmew-standardise-env restructure_dirs.sh

‎CMEW/flow.cylc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@
4141
SHARE_DATA_CDDS = ${CYLC_WORKFLOW_SHARE_DIR}/data/cdds
4242
ROOT_PROC_DIR = ${SHARE_DATA_CDDS}/proc
4343
ROOT_DATA_DIR = ${SHARE_DATA_CDDS}/cdds_data
44-
REQUEST_PATH = ${CYLC_WORKFLOW_SHARE_DIR}/etc/request.json
44+
REQUEST_PATH = ${CYLC_WORKFLOW_SHARE_DIR}/etc/request.cfg
4545
VARIABLES_PATH = ${CYLC_WORKFLOW_SHARE_DIR}/etc/variables.txt
4646

4747
[[ASSESSMENT_AREA]]

‎CMEW/opt/rose-suite-metoffice.conf‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,10 @@
22
# The LICENSE.md file contains full licensing details.
33

44
[template variables]
5-
CDDS_VERSION="2.5.5"
5+
CDDS_VERSION="3.2.0"
66
DRS_CMIP6="BADC"
77
DRS_OBS4MIPS="default"
8-
ESMVALTOOL_MODULE_NAME="scitools/community/esmvaltool/2.11.0"
9-
ROOTPATH_CMIP6="/project/champ/data/CMIP6"
8+
ESMVALTOOL_MODULE_NAME="scitools/community/esmvaltool/2.12.0"
9+
ROOTPATH_CMIP6="/data/users/managecmip/champ/CMIP6"
1010
ROOTPATH_OBS4MIPS="/data/users/esmval/ESMValTool/temporary/obs"
1111
SITE="metoffice"

‎pyproject.toml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,14 @@ exclude = '''
1515
)/
1616
)
1717
'''
18+
19+
# Configure pytest.
20+
[tool.pytest.ini_options]
21+
filterwarnings = [
22+
# Ignore deprecation warnings from webob, esmpy and pkg_resources:
23+
"ignore:'cgi' is deprecated and slated for removal in Python 3.13:DeprecationWarning",
24+
"ignore:Implicit None on return values is deprecated:DeprecationWarning",
25+
"ignore:Deprecated call to `pkg_resources.declare_namespace:DeprecationWarning",
26+
# Ignore user warnings from pybtex:
27+
"ignore:pkg_resources is deprecated as an API:UserWarning",
28+
]

0 commit comments

Comments
 (0)