From c9deb3b31312191b7844271616b216931d1e345b Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Fri, 14 Feb 2025 15:10:47 +1100 Subject: [PATCH 01/23] Added integration test script Added integration tests for hres_ic Added development environment --- .conda/env_dev.yml | 14 ++ .gitignore | 1 - integration/INTEGRATION_README.md | 80 ++++++++++ integration/integration_tests.sh | 142 ++++++++++++++++++ .../replace_landsurface_with_ERA5land_IC.py | 4 - 5 files changed, 236 insertions(+), 5 deletions(-) create mode 100644 .conda/env_dev.yml create mode 100644 integration/INTEGRATION_README.md create mode 100755 integration/integration_tests.sh diff --git a/.conda/env_dev.yml b/.conda/env_dev.yml new file mode 100644 index 0000000..38fbb06 --- /dev/null +++ b/.conda/env_dev.yml @@ -0,0 +1,14 @@ +channels: + - accessnri + - conda-forge + - coecms + - nodefaults + +dependencies: + - mule + - numpy <= 1.23.4 # https://stackoverflow.com/a/75148219/21024780 + - scitools-iris + - xarray + - versioneer + - pytest + - hypothesis \ No newline at end of file diff --git a/.gitignore b/.gitignore index 96ca83b..97b8d67 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ test_data/ .coverage -*.sh __pycache__/ *.py[cod] .ruff_cache/ diff --git a/integration/INTEGRATION_README.md b/integration/INTEGRATION_README.md new file mode 100644 index 0000000..bdfb3f9 --- /dev/null +++ b/integration/INTEGRATION_README.md @@ -0,0 +1,80 @@ +# Integration tests for replace_landsurface + +`integration_tests.sh` is a basic binary compatibility test script for replace_landsurface, +that compares the outputs of a specific version of the `replace_landsurface` package with their expected outputs. +The script warns the user if the outputs do not match their respective expected versions. + +The tests are designed to be run on Gadi within a Python environment where `replace_landsurface` is installed as a development package (for instructions refer to the Installation paragraph in the README). + + + + +## Data choices +Three types of reference data are available for use in the tests, called "full", +"intermediate", and "light". Each group of data contains a fields file, and +netCDF files produced from converting the fields file using various `um2nc` options. + +### Full +The "full" fields file is an output file from an ESM1.5 simulation. +Running tests with the "full" data is slower and more resource-intensive. + +### Intermediate (default) +The "intermediate" data contains a fields file, generated as a subset of the +the variables from the "full" data. These variables were selected to ensure +different portions of code within `um2nc` are used in the integration tests. +The included variables are: + +* m01s00i024 Surface temperature (a simple 2D field) +* m01s00i407 Pressure on model rho levels +* m01s00i408 Pressure on model theta levels +* m01s02i288 Aerosol optical thickness from biomass burning (a variable on pseudo_levels) +* m01s03i209 Eastward wind (tests hardcoded variable name changes) +* m01s03i321 Canopy water on tiles (a tiled variable) +* m01s05i216 Precipitation (a simple 2D field) +* m01s08i208 Soil moisture (land only data) +* m01s08i223 Soil moisture on soil levels +* m01s30i204 Temperature on PLEV grid (requires masking) +* m01s30i301 Heaviside (used for masking) + +### Light +The "light" data contains a minimal subset of variables from the "full" data +fields file, and can be used for faster but less in-depth testing. It includes: + +* m01s30i204 Temperature on PLEV grid +* m01s05i216 Precipitation + +### Comparison data +For each of the above data choices, the following netCDF variants were produced: + +* `mask`: produced with the `--nohist` flag only. +* `nomask`: produced with the `--nomask` and `--nohist` flags. +* `hist`: produced with no flags. These files will have a conversion datestamp +in their history attribute. + +## Data versions +The `um2nc` version to compare against can be selected with the `-v` flag. +If omitted, the tests will be performed against the latest released version. + +Available versions for comparison are: +* 0 + +### Version `0` +Version `0` netCDF outputs were created using the `um2netcdf.py` script available +prior to the development of `um2nc`: https://github.com/ACCESS-NRI/um2nc-standalone/blob/f62105b45eb39d2beed5a7ac71f439ff90f0f00c/src/um2netcdf.py + +The conversion was performed within the following `payu1.1.5` environment, active on Gadi: +https://github.com/ACCESS-NRI/payu-condaenv/releases/tag/1.1.5 + +All test data is located in `/g/data/vk83/testing/data/um2nc/integration-tests`. diff --git a/integration/integration_tests.sh b/integration/integration_tests.sh new file mode 100755 index 0000000..78e6252 --- /dev/null +++ b/integration/integration_tests.sh @@ -0,0 +1,142 @@ +#!/bin/bash + +# Basic binary compatibility test script for replace_landsurface. +# See INTEGRATION_README.md for details on test usage, data and options. + +TEST_DATA_DIR=/g/data/tm70/dm5220/projects/replace_landsurface/test_data +INPUT_DIR=$TEST_DATA_DIR/input_data +OUTPUT_DIR=$TEST_DATA_DIR/expected_outputs +DRIVING_DATA_DIR=$TEST_DATA_DIR/driving_data +CLEAN_OUTPUT=true + +# Create a temporary work directory +WORK_DIR=$(mktemp -d) +echo -e "Work directory: $WORK_DIR\n" +# Trap signals to clean up WORK directory depending on exit status +functrap() { + code="$?" + if ([ "$code" -eq 0 ] && $CLEAN_OUTPUT) || [ "$code" -eq 2 ]; then + rm -rf "$WORK_DIR" + echo "Work directory cleaned up." + fi +} +# Separate the cases when the script is interrupted, from the cases when it's not +trap "exit 2" SIGHUP SIGINT SIGQUIT SIGILL SIGABRT SIGTERM +trap functrap EXIT + +#Set up the work directory as a copy of the test data directory +echo "Setting up work directory..." +cp -r $INPUT_DIR/* $WORK_DIR +cd $WORK_DIR + + +function usage { + cat << EOF +Basic binary compatibility test script for 'replace_landsurface'. +Checks that the outputs of the 'replace_landsurface' package entry points correspond to the expected outputs. + +Usage: regression_tests.sh [-k/--keep] + +Options: +-k, --keep Keep output data upon test completion. + If absent, output data will only be kept for failed tests. +EOF +} + +while getopts ":-:hk" opt; do + case ${opt} in + -) + case ${OPTARG} in + help) + usage + exit 0 + ;; + keep) + CLEAN_OUTPUT=false + ;; + *) + echo "Invalid option: \"--${OPTARG}\"." >&2 + usage + exit 1 + ;; + esac + ;; + h) + usage + exit 0 + ;; + k) + CLEAN_OUTPUT=true + ;; + \?) + echo "Invalid option: \"-${OPTARG}\"." >&2 + usage + exit 1 + ;; + esac +done + +# Check that no additional arguments were passed. +if [[ -n "${@:$OPTIND:1}" ]]; then + echo "Invalid positional argument: \"${@:$OPTIND:1}\"." >&2 + exit 1 +fi + +# # ----------------------------------------------------------------- +# # Run the tests +# # ----------------------------------------------------------------- +echo "Running tests..." + +export ROSE_DATA=$DRIVING_DATA_DIR +run_command() { + MASK=$WORK_DIR/$test_dir/mask + FILE=$WORK_DIR/$test_dir/file + HRES_IC=$WORK_DIR/$test_dir/hres_ic + eval "$entry_point --mask $MASK --file ${FILE}.tmp --start $START --hres_ic $HRES_IC --type $TYPE" +} +compare() { + cmp $WORK_DIR/$test_dir/file $OUTPUT_DIR/$test_dir + if [ $? -ne 0 ]; then + echo "Test ${test_dir#test} failed." + exit 1 + fi +} +# # ----------------------------------------------------------------- +# Test hres_ic +entry_point=hres_ic + +# Test 1: 'era5land' +# MASK=/scratch/tm70/cbe563/cylc-run/u-dg767/share/data/ancils/Lismore/d1000/qrparm.mask +# FILE=g/data/tm70/cbe563/test_replace_land_surface/ERA5LAND/GAL9_astart +# HRES_IC=NOT_SET +test_dir=test_1 +TYPE=era5land +START=202202260000 + +echo "### Test 1: hres_ic, type 'era5land' ###" +run_command +compare + +# # Test 2: 'barra' +# MASK=/scratch/tm70/cbe563/cylc-run/u-dg767.b/share/data/ancils/Lismore/d1100/qrparm.mask +# FILE=/g/data/tm70/cbe563/test_replace_land_surface/BARRAR2/GAL9_astart +# HRES_IC=NOT_SET +test_dir=test_2 +TYPE=barra +START=202008090000 + +echo "### Test 2: hres_ic, type 'barra' ###" +run_command +compare + +# # Test 2: 'barra' +# MASK=/scratch/tm70/cbe563/cylc-run/u-dg767/share/data/ancils/Lismore/d0198/qrparm.mask +# FILE=/g/data/tm70/cbe563/test_replace_land_surface/ASTART/RAL3P2_astart +# HRES_IC=/scratch/tm70/cbe563/cylc-run/u-dg768.worked/share/cycle/20220226T0000Z/Lismore/d0198/RAL3P2/ics/RAL3P2_astart +test_dir=test_3 +TYPE=astart +START=202112310000 + +echo "### Test 3: hres_ic, type 'astart' ###" +run_command +compare \ No newline at end of file diff --git a/src/replace_landsurface/replace_landsurface_with_ERA5land_IC.py b/src/replace_landsurface/replace_landsurface_with_ERA5land_IC.py index 8d3bb92..b744279 100755 --- a/src/replace_landsurface/replace_landsurface_with_ERA5land_IC.py +++ b/src/replace_landsurface/replace_landsurface_with_ERA5land_IC.py @@ -215,10 +215,6 @@ def swap_land_era5land(mask_fullpath, ic_file_fullpath, ic_date): The file is replaced with a version of itself holding the higher-resolution data. """ - - # create name of file to be replaced - ic_file = ic_file_fullpath.parts[-1].replace('.tmp', '') - # create date/time useful information #print(ic_date) yyyy = ic_date[0:4] From 284cf26ebd585458c7fcf6a01203a8fc9f16a432 Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Sat, 22 Feb 2025 09:42:45 +1100 Subject: [PATCH 02/23] Added parallelization for tests with dynamic capture of PIDs and exit statuses --- integration/integration_tests.sh | 45 +++++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/integration/integration_tests.sh b/integration/integration_tests.sh index 78e6252..2802668 100755 --- a/integration/integration_tests.sh +++ b/integration/integration_tests.sh @@ -101,7 +101,12 @@ compare() { exit 1 fi } +run_test() { + run_command + compare +} # # ----------------------------------------------------------------- +declare -A pids # Associative array (similar to a dictionary in Python) # Test hres_ic entry_point=hres_ic @@ -114,8 +119,7 @@ TYPE=era5land START=202202260000 echo "### Test 1: hres_ic, type 'era5land' ###" -run_command -compare +run_test & pids[1]+=$! # # Test 2: 'barra' # MASK=/scratch/tm70/cbe563/cylc-run/u-dg767.b/share/data/ancils/Lismore/d1100/qrparm.mask @@ -126,8 +130,7 @@ TYPE=barra START=202008090000 echo "### Test 2: hres_ic, type 'barra' ###" -run_command -compare +run_test & pids[2]+=$! # # Test 2: 'barra' # MASK=/scratch/tm70/cbe563/cylc-run/u-dg767/share/data/ancils/Lismore/d0198/qrparm.mask @@ -138,5 +141,35 @@ TYPE=astart START=202112310000 echo "### Test 3: hres_ic, type 'astart' ###" -run_command -compare \ No newline at end of file +run_test & pids[3]+=$! + + +# Capture the exit status of each background processes dynamically +declare -A exit_statuses # Associative array (similar to a dictionary in Python) +for pid in "${pids[@]}"; do + # Find which test the pid corresponds to + for key in "${!pids[@]}"; do + if [[ ${pids[$key]} == $pid ]]; then + test_num=$key + break + fi + done + wait $pid # Waits for the next job in the list to finish (not necessarily in order because it's used within a loop) + exit_status=$? + exit_statuses[$test_num]=$exit_status +done + +# Exit with a status 1 if any of the tests failed +exit_value=0 +for ind in ${!exit_statuses[@]}; do + if [ ${exit_statuses[$ind]} -ne 0 ]; then + echo "Test $ind failed." + exit_value=1 + fi +done + +if [ $exit_value -eq 0 ]; then + echo "All tests passed." +else + exit 1 +fi \ No newline at end of file From e7329856a3f7d7dad8ff958bd163eefe95b61fcc Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Sat, 22 Feb 2025 09:48:17 +1100 Subject: [PATCH 03/23] Redirected tests STDOUT to /dev/null --- integration/integration_tests.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/integration/integration_tests.sh b/integration/integration_tests.sh index 2802668..93e8bdf 100755 --- a/integration/integration_tests.sh +++ b/integration/integration_tests.sh @@ -9,6 +9,8 @@ OUTPUT_DIR=$TEST_DATA_DIR/expected_outputs DRIVING_DATA_DIR=$TEST_DATA_DIR/driving_data CLEAN_OUTPUT=true +declare -A pids # Associative array (similar to a dictionary in Python) + # Create a temporary work directory WORK_DIR=$(mktemp -d) echo -e "Work directory: $WORK_DIR\n" @@ -106,7 +108,6 @@ run_test() { compare } # # ----------------------------------------------------------------- -declare -A pids # Associative array (similar to a dictionary in Python) # Test hres_ic entry_point=hres_ic @@ -119,7 +120,7 @@ TYPE=era5land START=202202260000 echo "### Test 1: hres_ic, type 'era5land' ###" -run_test & pids[1]+=$! +run_test > /dev/null & pids[1]+=$! # # Test 2: 'barra' # MASK=/scratch/tm70/cbe563/cylc-run/u-dg767.b/share/data/ancils/Lismore/d1100/qrparm.mask @@ -130,7 +131,7 @@ TYPE=barra START=202008090000 echo "### Test 2: hres_ic, type 'barra' ###" -run_test & pids[2]+=$! +run_test > /dev/null & pids[2]+=$! # # Test 2: 'barra' # MASK=/scratch/tm70/cbe563/cylc-run/u-dg767/share/data/ancils/Lismore/d0198/qrparm.mask @@ -141,7 +142,7 @@ TYPE=astart START=202112310000 echo "### Test 3: hres_ic, type 'astart' ###" -run_test & pids[3]+=$! +run_test > /dev/null & pids[3]+=$! # Capture the exit status of each background processes dynamically From 37c3ff0906db81c6a53048af7edcff316828cc5e Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Sat, 22 Feb 2025 10:24:32 +1100 Subject: [PATCH 04/23] Updated trap function. Updated compare function. Updated paths --- integration/integration_tests.sh | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/integration/integration_tests.sh b/integration/integration_tests.sh index 93e8bdf..6d0a45e 100755 --- a/integration/integration_tests.sh +++ b/integration/integration_tests.sh @@ -3,7 +3,7 @@ # Basic binary compatibility test script for replace_landsurface. # See INTEGRATION_README.md for details on test usage, data and options. -TEST_DATA_DIR=/g/data/tm70/dm5220/projects/replace_landsurface/test_data +TEST_DATA_DIR=/g/data/vk83/testing/data/replace_landsurface/integration_tests INPUT_DIR=$TEST_DATA_DIR/input_data OUTPUT_DIR=$TEST_DATA_DIR/expected_outputs DRIVING_DATA_DIR=$TEST_DATA_DIR/driving_data @@ -16,14 +16,19 @@ WORK_DIR=$(mktemp -d) echo -e "Work directory: $WORK_DIR\n" # Trap signals to clean up WORK directory depending on exit status functrap() { - code="$?" - if ([ "$code" -eq 0 ] && $CLEAN_OUTPUT) || [ "$code" -eq 2 ]; then + exit_code="$?" + if [ "$exit_code" -eq 0 ] && $CLEAN_OUTPUT; then # Job completed successfully and CLEAN_OUTPUT is true rm -rf "$WORK_DIR" echo "Work directory cleaned up." + elif [ "$exit_code" -eq 2 ]; then # Job was terminated preventively (did not fail) + kill -9 $(jobs -p) &> /dev/null # Kill all background jobs + rm -rf "$WORK_DIR" + echo "Script was terminated preventively." + echo "Work directory cleaned up. Background jobs killed." fi } # Separate the cases when the script is interrupted, from the cases when it's not -trap "exit 2" SIGHUP SIGINT SIGQUIT SIGILL SIGABRT SIGTERM +trap "exit 2" SIGHUP SIGINT SIGQUIT SIGILL SIGABRT SIGTERM SIGSTOP trap functrap EXIT #Set up the work directory as a copy of the test data directory @@ -97,9 +102,10 @@ run_command() { eval "$entry_point --mask $MASK --file ${FILE}.tmp --start $START --hres_ic $HRES_IC --type $TYPE" } compare() { - cmp $WORK_DIR/$test_dir/file $OUTPUT_DIR/$test_dir + test_num=${test_dir#test_} + cmp $WORK_DIR/$test_dir/file $OUTPUT_DIR/output_${test_num} if [ $? -ne 0 ]; then - echo "Test ${test_dir#test} failed." + echo "Test $test_num failed." exit 1 fi } @@ -144,7 +150,6 @@ START=202112310000 echo "### Test 3: hres_ic, type 'astart' ###" run_test > /dev/null & pids[3]+=$! - # Capture the exit status of each background processes dynamically declare -A exit_statuses # Associative array (similar to a dictionary in Python) for pid in "${pids[@]}"; do From a1e85dd07e16cc6d60a7d0de8348961e286d2d86 Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Sat, 22 Feb 2025 16:37:41 +1100 Subject: [PATCH 05/23] Minor fix to a comment --- integration/integration_tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration/integration_tests.sh b/integration/integration_tests.sh index 6d0a45e..42851c2 100755 --- a/integration/integration_tests.sh +++ b/integration/integration_tests.sh @@ -119,7 +119,7 @@ entry_point=hres_ic # Test 1: 'era5land' # MASK=/scratch/tm70/cbe563/cylc-run/u-dg767/share/data/ancils/Lismore/d1000/qrparm.mask -# FILE=g/data/tm70/cbe563/test_replace_land_surface/ERA5LAND/GAL9_astart +# FILE=/g/data/tm70/cbe563/test_replace_land_surface/ERA5LAND/GAL9_astart # HRES_IC=NOT_SET test_dir=test_1 TYPE=era5land @@ -139,7 +139,7 @@ START=202008090000 echo "### Test 2: hres_ic, type 'barra' ###" run_test > /dev/null & pids[2]+=$! -# # Test 2: 'barra' +# # Test 3: 'astart' # MASK=/scratch/tm70/cbe563/cylc-run/u-dg767/share/data/ancils/Lismore/d0198/qrparm.mask # FILE=/g/data/tm70/cbe563/test_replace_land_surface/ASTART/RAL3P2_astart # HRES_IC=/scratch/tm70/cbe563/cylc-run/u-dg768.worked/share/cycle/20220226T0000Z/Lismore/d0198/RAL3P2/ics/RAL3P2_astart From 670197093cd1ba73cbfca04a489a797910229241 Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Mon, 24 Feb 2025 09:08:28 +1100 Subject: [PATCH 06/23] Updated integration README --- integration/INTEGRATION_README.md | 88 +++++++------------------------ 1 file changed, 18 insertions(+), 70 deletions(-) diff --git a/integration/INTEGRATION_README.md b/integration/INTEGRATION_README.md index bdfb3f9..f568606 100644 --- a/integration/INTEGRATION_README.md +++ b/integration/INTEGRATION_README.md @@ -1,80 +1,28 @@ # Integration tests for replace_landsurface -`integration_tests.sh` is a basic binary compatibility test script for replace_landsurface, -that compares the outputs of a specific version of the `replace_landsurface` package with their expected outputs. -The script warns the user if the outputs do not match their respective expected versions. +`integration_tests.sh` is a basic binary compatibility test script for the replace_landsurface package. -The tests are designed to be run on Gadi within a Python environment where `replace_landsurface` is installed as a development package (for instructions refer to the Installation paragraph in the README). - - - - -## Data choices -Three types of reference data are available for use in the tests, called "full", -"intermediate", and "light". Each group of data contains a fields file, and -netCDF files produced from converting the fields file using various `um2nc` options. - -### Full -The "full" fields file is an output file from an ESM1.5 simulation. -Running tests with the "full" data is slower and more resource-intensive. +The script executes various tests in parallel, for all the entry points of the package, using different options to test multiple cases. -### Intermediate (default) -The "intermediate" data contains a fields file, generated as a subset of the -the variables from the "full" data. These variables were selected to ensure -different portions of code within `um2nc` are used in the integration tests. -The included variables are: +All multiple tests are executed to completion, even if any of them fails. +When all the tests are completed, the script fails if any of the tested returned a non-zero exit code, and a message is printed with information about the failed test. -* m01s00i024 Surface temperature (a simple 2D field) -* m01s00i407 Pressure on model rho levels -* m01s00i408 Pressure on model theta levels -* m01s02i288 Aerosol optical thickness from biomass burning (a variable on pseudo_levels) -* m01s03i209 Eastward wind (tests hardcoded variable name changes) -* m01s03i321 Canopy water on tiles (a tiled variable) -* m01s05i216 Precipitation (a simple 2D field) -* m01s08i208 Soil moisture (land only data) -* m01s08i223 Soil moisture on soil levels -* m01s30i204 Temperature on PLEV grid (requires masking) -* m01s30i301 Heaviside (used for masking) - -### Light -The "light" data contains a minimal subset of variables from the "full" data -fields file, and can be used for faster but less in-depth testing. It includes: - -* m01s30i204 Temperature on PLEV grid -* m01s05i216 Precipitation - -### Comparison data -For each of the above data choices, the following netCDF variants were produced: - -* `mask`: produced with the `--nohist` flag only. -* `nomask`: produced with the `--nomask` and `--nohist` flags. -* `hist`: produced with no flags. These files will have a conversion datestamp -in their history attribute. +The tests are designed to be run on Gadi within a Python environment where `replace_landsurface` is installed as a development package (for instructions refer to the Installation paragraph in the README). -## Data versions -The `um2nc` version to compare against can be selected with the `-v` flag. -If omitted, the tests will be performed against the latest released version. +Usage: + regression_tests.sh [-h] [--keep] -Available versions for comparison are: -* 0 +Options: + -h, --help Print this usage information and exit. + -k, --keep Force output data to be kept upon test completion. + The default behaviour is to keep the output data only for failed test sessions and delete it if all tests pass. -### Version `0` -Version `0` netCDF outputs were created using the `um2netcdf.py` script available -prior to the development of `um2nc`: https://github.com/ACCESS-NRI/um2nc-standalone/blob/f62105b45eb39d2beed5a7ac71f439ff90f0f00c/src/um2netcdf.py +All test data is located in `/g/data/vk83/testing/data/replace_landsurface/integration_tests`. -The conversion was performed within the following `payu1.1.5` environment, active on Gadi: -https://github.com/ACCESS-NRI/payu-condaenv/releases/tag/1.1.5 +### Tests performed -All test data is located in `/g/data/vk83/testing/data/um2nc/integration-tests`. +- Test 1: hres_ic with `--type era5land` +- Test 2: hres_ic with `--type barra` +- Test 3: hres_ic with `--type astart` +- Test 4: hres_eccb with `--type era5land` (CURRENTLY NOT AVAILABLE) +- Test 5: hres_eccb with `--type barra` (CURRENTLY NOT AVAILABLE) \ No newline at end of file From fb5ffa8c7c45bedc4a6359cb2edc22c557ba9f8f Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Mon, 24 Feb 2025 09:27:15 +1100 Subject: [PATCH 07/23] Updated README --- README.md | 40 +++++++++++++++++++++++++++++--- integration/integration_tests.sh | 3 ++- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 18318dd..42548af 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,39 @@ -[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)](https://github.com/pre-commit/pre-commit) - # replace_landsurface -This repository contains Python scripts for use by Regional Nesting Suites to replace specific land surface fields in static input data. +## About + +`replace_landsurface` is a `Python` utility to be used within ACCESS-NRI versions of the Regional Nesting Suites to replace specific land surface initial/boundary conditions. + + +## Development/Testing instructions +For development/testing, it is recommended to install `replace_landsurface` as a development package within a `micromamba`/`conda` testing environment. + +### Clone replace_landsurface GitHub repo +``` +git clone git@github.com:ACCESS-NRI/replace_landsurface.git +``` + +### Create a micromamba/conda testing environment +> [!TIP] +> In the following instructions `micromamba` can be replaced with `conda`. + +``` +cd replace_landsurface +micromamba env create -n replace_landsurface_dev --file .conda/env_dev.yml +micromamba activate replace_landsurface_dev +``` + +### Install replace_landsurface as a development package +``` +pip install --no-deps --no-build-isolation -e . +``` + +### Running the tests + +#### Integration tests +To manually run the integration tests, from the `replace_landsurface` directory, you can: +1. Activate your [micromamba/conda testing environment](#create-a-micromamba-conda-testing-environment) +2. Run the following command: + ``` + bash integration/integration_tests.sh + ``` \ No newline at end of file diff --git a/integration/integration_tests.sh b/integration/integration_tests.sh index 42851c2..da731b4 100755 --- a/integration/integration_tests.sh +++ b/integration/integration_tests.sh @@ -31,11 +31,12 @@ functrap() { trap "exit 2" SIGHUP SIGINT SIGQUIT SIGILL SIGABRT SIGTERM SIGSTOP trap functrap EXIT +start_time=$(date +%s) #Set up the work directory as a copy of the test data directory echo "Setting up work directory..." cp -r $INPUT_DIR/* $WORK_DIR cd $WORK_DIR - +echo "Elapsed time for data copy: $(($(date +%s) - start_time)) seconds" function usage { cat << EOF From 731e0bcacf26e4feb5922108cdda3ad8782e91da Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Mon, 24 Feb 2025 18:12:02 +1100 Subject: [PATCH 08/23] Added dependencies to dev environment --- .conda/env_dev.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.conda/env_dev.yml b/.conda/env_dev.yml index 38fbb06..9580206 100644 --- a/.conda/env_dev.yml +++ b/.conda/env_dev.yml @@ -11,4 +11,6 @@ dependencies: - xarray - versioneer - pytest + - pytest-cov + - pytest-xdist - hypothesis \ No newline at end of file From 2f9785669f4c95502d78791124b6038ef52426bb Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Tue, 25 Feb 2025 18:01:01 +1100 Subject: [PATCH 09/23] Added tests/test_integration.py to be run with pytest. Before the actual test run, there is a setup step that sets up the work directory and data paths Currently the tests are not meant to be run in parallel yet, as the setup should be re-structured to be run only once (before the test run) and not for each parallel worker. --- tests/integration/test_integration.py | 122 ++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 tests/integration/test_integration.py diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py new file mode 100644 index 0000000..834cca5 --- /dev/null +++ b/tests/integration/test_integration.py @@ -0,0 +1,122 @@ +import filecmp +import os +import pytest +import shutil +import socket +import tempfile +from unittest.mock import patch +from datetime import datetime + +# If not on Gadi, fail the test because the test data is not available +GADI_HOSTNAME = "gadi.nci.org.au" +hostname = socket.gethostname() +if not hostname.endswith(GADI_HOSTNAME): + raise EnvironmentError( + f"Test not supported to be run on {hostname}. This test must be run on Gadi (gadi.nci.org.au)." + ) + +############################################ +## === Integration tests setup === ## +############################################ +print("\n## === Integration tests setup started === ##") +TEST_DATA_DIR = '/g/data/vk83/testing/data/replace_landsurface/integration_tests' +INPUT_DIR = os.path.join(TEST_DATA_DIR,'input_data') +OUTPUT_DIR = os.path.join(TEST_DATA_DIR,'expected_outputs') +DRIVING_DATA_DIR = os.path.join(TEST_DATA_DIR,'driving_data') +# Set the ROSE_DATA environment variable to the driving data directory +os.environ["ROSE_DATA"] = "/g/data/vk83/testing/data/replace_landsurface/integration_tests/driving_data" +from replace_landsurface import hres_ic # importing here because we need to set the ROSE_DATA env variable before importing # noqa + +# Set up working directory path' +working_dir_prefix='replace_landsurface_integration_tests_' +working_dir_date=datetime.now().strftime("%Y%m%d%H%M%S") +WORKING_DIR = os.path.join(tempfile.gettempdir(), working_dir_prefix + working_dir_date) +# Copy input data to the working directory +shutil.copytree(INPUT_DIR, WORKING_DIR) +# Change current working directory to the working directory +os.chdir(WORKING_DIR) +print("## === Integration tests setup complete! === ##") +############################################ +## === Integration tests === ## +############################################ +# Suppress warnings +pytestmark = pytest.mark.filterwarnings("ignore::Warning") + +def get_error_msg(num, output, expected_output): + return f"Test {num}: Test output '{output}' does not match the expected output '{expected_output}'!" +# Define the command-line arguments for the tests +def get_test_args(num, start, _type): + test_dir = f'test_{num}' + _hres_ic = os.path.join(WORKING_DIR,test_dir,'hres_ic') if _type == 'astart' else 'NOT_USED' + return [ + "script_name", + '--mask', + os.path.join(WORKING_DIR,test_dir,'mask'), + '--file', + os.path.join(WORKING_DIR,test_dir,'file'+'.tmp'), + '--start', + start, + '--type', + _type, + '--hres_ic', + _hres_ic, + ] + +def get_output_path(num): + return os.path.join(WORKING_DIR,f'test_{num}','file') + +def get_expected_output_path(num): + return os.path.join(OUTPUT_DIR,f'output_{num}') + +# Set the ROSE_DATA environment variable to the driving data directory in all tests +@pytest.fixture(autouse=True) +def mock_rose_data(monkeypatch): + monkeypatch.setenv("ROSE_DATA", DRIVING_DATA_DIR) + +@patch("sys.argv", get_test_args(1,'202202260000','era5land')) +def test_hres_ic_era5land(): + """ + Test the hres_ic entry point with '--type era5land' + """ + num=1 + hres_ic.main() + output = get_output_path(num) + expected_output = get_expected_output_path(num) + # # Compare the output file with the expected output + assert filecmp.cmp(output, expected_output), get_error_msg(num, output, expected_output) + +@patch("sys.argv", get_test_args(2,'202008090000','barra')) +def test_hres_ic_barra(): + """ + Test the hres_ic entry point with '--type barra' + """ + num=2 + hres_ic.main() + output = get_output_path(num) + expected_output = get_expected_output_path(num) + # # Compare the output file with the expected output + assert filecmp.cmp(output, expected_output), get_error_msg(num, output, expected_output) + +@patch("sys.argv", get_test_args(3,'202112310000','astart')) +def test_hres_ic_astart(): + """ + Test the hres_ic entry point with '--type astart' + """ + num=3 + hres_ic.main() + output = get_output_path(num) + expected_output = get_expected_output_path(num) + # # Compare the output file with the expected output + assert filecmp.cmp(output, expected_output), get_error_msg(num, output, expected_output) + +def test_hres_eccb_era5land(): + """ + Test the hres_eccb entry point with '--type era5land' + """ + pass + +def test_hres_eccb_barra(): + """ + Test the hres_eccb entry point with '--type barra' + """ + pass \ No newline at end of file From 8d225ced613fbb1bcf9308fcb98f79161908069c Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Thu, 27 Feb 2025 00:00:02 +1100 Subject: [PATCH 10/23] Re-factored test to: - Mock the shutil.move function so the outputs can be sent to a custom path and there is no need for copying the input data - Use fixtures - Use mark.parameterize - Use built-in tmp_path_factory fixture - Enable parallelized tests Tests can now be run with 'pytest -n | auto' to run in parallel. --- tests/integration/test_integration.py | 192 ++++++++++++++++---------- 1 file changed, 119 insertions(+), 73 deletions(-) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 834cca5..a6bc479 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -1,113 +1,158 @@ +import contextlib import filecmp import os -import pytest import shutil import socket -import tempfile from unittest.mock import patch -from datetime import datetime +import pytest -# If not on Gadi, fail the test because the test data is not available +# If not on Gadi, skip the tests because the test data is not available GADI_HOSTNAME = "gadi.nci.org.au" hostname = socket.gethostname() -if not hostname.endswith(GADI_HOSTNAME): - raise EnvironmentError( - f"Test not supported to be run on {hostname}. This test must be run on Gadi (gadi.nci.org.au)." - ) +# Marker to skip tests if not on Gadi +skip_marker = pytest.mark.skipif( + not hostname.endswith(GADI_HOSTNAME), + reason=f"Skipping integration tests because they cannot be executed on {hostname}.\n" + "Integration tests are specifically designed to run on Gadi (gadi.nci.org.au).", +) +# Marker to suppress warnings +warning_marker = pytest.mark.filterwarnings("ignore::Warning") +# Apply the markers to all tests in this file +pytestmark = [skip_marker, warning_marker] ############################################ ## === Integration tests setup === ## ############################################ -print("\n## === Integration tests setup started === ##") -TEST_DATA_DIR = '/g/data/vk83/testing/data/replace_landsurface/integration_tests' -INPUT_DIR = os.path.join(TEST_DATA_DIR,'input_data') -OUTPUT_DIR = os.path.join(TEST_DATA_DIR,'expected_outputs') -DRIVING_DATA_DIR = os.path.join(TEST_DATA_DIR,'driving_data') +TEST_DATA_DIR = "/g/data/vk83/testing/data/replace_landsurface/integration_tests" +INPUT_DIR = os.path.join(TEST_DATA_DIR, "input_data") +OUTPUT_DIR = os.path.join(TEST_DATA_DIR, "expected_outputs") +DRIVING_DATA_DIR = os.path.join(TEST_DATA_DIR, "driving_data") # Set the ROSE_DATA environment variable to the driving data directory -os.environ["ROSE_DATA"] = "/g/data/vk83/testing/data/replace_landsurface/integration_tests/driving_data" -from replace_landsurface import hres_ic # importing here because we need to set the ROSE_DATA env variable before importing # noqa - -# Set up working directory path' -working_dir_prefix='replace_landsurface_integration_tests_' -working_dir_date=datetime.now().strftime("%Y%m%d%H%M%S") -WORKING_DIR = os.path.join(tempfile.gettempdir(), working_dir_prefix + working_dir_date) -# Copy input data to the working directory -shutil.copytree(INPUT_DIR, WORKING_DIR) -# Change current working directory to the working directory -os.chdir(WORKING_DIR) -print("## === Integration tests setup complete! === ##") +os.environ["ROSE_DATA"] = DRIVING_DATA_DIR +from replace_landsurface import ( + hres_ic, +) # importing here because we need to set the ROSE_DATA env variable before importing # noqa + + ############################################ ## === Integration tests === ## ############################################ -# Suppress warnings -pytestmark = pytest.mark.filterwarnings("ignore::Warning") - -def get_error_msg(num, output, expected_output): - return f"Test {num}: Test output '{output}' does not match the expected output '{expected_output}'!" -# Define the command-line arguments for the tests def get_test_args(num, start, _type): - test_dir = f'test_{num}' - _hres_ic = os.path.join(WORKING_DIR,test_dir,'hres_ic') if _type == 'astart' else 'NOT_USED' + test_dir = f"test_{num}" + _hres_ic = ( + os.path.join(INPUT_DIR, test_dir, "hres_ic") + if _type == "astart" + else "NOT_USED" + ) return [ "script_name", - '--mask', - os.path.join(WORKING_DIR,test_dir,'mask'), - '--file', - os.path.join(WORKING_DIR,test_dir,'file'+'.tmp'), - '--start', + "--mask", + os.path.join(INPUT_DIR, test_dir, "mask"), + "--file", + os.path.join(INPUT_DIR, test_dir, "file" + ".tmp"), + "--start", start, - '--type', + "--type", _type, - '--hres_ic', + "--hres_ic", _hres_ic, ] -def get_output_path(num): - return os.path.join(WORKING_DIR,f'test_{num}','file') -def get_expected_output_path(num): - return os.path.join(OUTPUT_DIR,f'output_{num}') +@pytest.fixture +def mock_sys_argv(): + @contextlib.contextmanager + def _mock_sys_argv(num, start, _type): + with patch("sys.argv", get_test_args(num, start, _type)): + yield mock_sys_argv + + return _mock_sys_argv + + +@pytest.fixture() +def get_error_msg(): + def _get_error_msg(num, output, expected_output): + return f"Test {num}: Test output '{output}' does not match the expected output '{expected_output}'!" + + return _get_error_msg + + +@pytest.fixture(scope="module") +def working_dir(tmp_path_factory): + return tmp_path_factory.mktemp("replace_landsurface_integration_tests") + + +@pytest.fixture() +def get_output_path(working_dir): + def _get_output_path(num): + return os.path.join(working_dir, f"output_{num}") + + return _get_output_path + + +@pytest.fixture() +def get_expected_output_path(): + def _get_expected_output_path(num): + return os.path.join(OUTPUT_DIR, f"output_{num}") + + return _get_expected_output_path + # Set the ROSE_DATA environment variable to the driving data directory in all tests @pytest.fixture(autouse=True) def mock_rose_data(monkeypatch): monkeypatch.setenv("ROSE_DATA", DRIVING_DATA_DIR) -@patch("sys.argv", get_test_args(1,'202202260000','era5land')) -def test_hres_ic_era5land(): - """ - Test the hres_ic entry point with '--type era5land' - """ - num=1 - hres_ic.main() - output = get_output_path(num) - expected_output = get_expected_output_path(num) - # # Compare the output file with the expected output - assert filecmp.cmp(output, expected_output), get_error_msg(num, output, expected_output) -@patch("sys.argv", get_test_args(2,'202008090000','barra')) -def test_hres_ic_barra(): - """ - Test the hres_ic entry point with '--type barra' - """ - num=2 - hres_ic.main() - output = get_output_path(num) - expected_output = get_expected_output_path(num) - # # Compare the output file with the expected output - assert filecmp.cmp(output, expected_output), get_error_msg(num, output, expected_output) +@pytest.fixture(scope="module") +def original_shutil_move(): + return shutil.move -@patch("sys.argv", get_test_args(3,'202112310000','astart')) -def test_hres_ic_astart(): + +@pytest.fixture() +def new_shutil_move(original_shutil_move, get_output_path, get_expected_output_path): + def _new_shutil_move(num): + def _wrapper(src, dst, **kwargs): + output_path = get_output_path(num) + return original_shutil_move(src=src, dst=output_path, **kwargs) + + return _wrapper + + return _new_shutil_move + + +@pytest.mark.parametrize( + "num, start, _type", + [ + (1, "202202260000", "era5land"), + (2, "202008090000", "barra"), + (3, "202112310000", "astart"), + ], + ids=["hres_ic_era5land", "hres_ic_barra", "hres_ic_astart"], +) +def test_hres_ic_era5land( + new_shutil_move, + get_output_path, + get_expected_output_path, + get_error_msg, + mock_sys_argv, + num, + start, + _type, +): """ - Test the hres_ic entry point with '--type astart' + Test the hres_ic entry point with '--type era5land' """ - num=3 - hres_ic.main() + with mock_sys_argv(num, start, _type): + with patch("shutil.move", side_effect=new_shutil_move(num)): + hres_ic.main() output = get_output_path(num) expected_output = get_expected_output_path(num) # # Compare the output file with the expected output - assert filecmp.cmp(output, expected_output), get_error_msg(num, output, expected_output) + assert filecmp.cmp(output, expected_output), get_error_msg( + num, output, expected_output + ) + def test_hres_eccb_era5land(): """ @@ -115,8 +160,9 @@ def test_hres_eccb_era5land(): """ pass + def test_hres_eccb_barra(): """ Test the hres_eccb entry point with '--type barra' """ - pass \ No newline at end of file + pass From ad7ea52077d6a4ef7100fa4bfa4cb218fc0f5001 Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Thu, 27 Feb 2025 11:46:32 +1100 Subject: [PATCH 11/23] Added tests for hres_eccb entry point --- .../replace_landsurface_with_BARRA2R_IC.py | 2 - tests/integration/test_integration.py | 52 ++++++++++++++----- 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py b/src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py index a420754..4f64835 100755 --- a/src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py +++ b/src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py @@ -152,7 +152,6 @@ def get_BARRA_nc_data(ncfname, FIELDN, wanted_dt, NLAYERS, bounds): sys.exit(1) d.close() - return data.data @@ -202,7 +201,6 @@ def swap_land_barra(mask_fullpath, ec_cb_file_fullpath, ic_date): indir = os.path.join(BARRA_DIR, '1hr',BARRA_FIELDN, 'latest') barra_files = glob(os.path.join(indir, BARRA_FIELDN + '*' + yyyy + mm + '*nc')) barra_fname = indir + '/' + barra_files[0].split('/')[-1] - # Work out the grid bounds using the surface temperature file bounds = bounding_box(barra_fname, mask_fullpath.as_posix(), "land_binary_mask") diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index a6bc479..1cba67c 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -128,9 +128,13 @@ def _wrapper(src, dst, **kwargs): (2, "202008090000", "barra"), (3, "202112310000", "astart"), ], - ids=["hres_ic_era5land", "hres_ic_barra", "hres_ic_astart"], + ids=[ + "hres_ic_era5land", + "hres_ic_barra", + "hres_ic_astart", + ], ) -def test_hres_ic_era5land( +def test_hres_ic( new_shutil_move, get_output_path, get_expected_output_path, @@ -141,7 +145,7 @@ def test_hres_ic_era5land( _type, ): """ - Test the hres_ic entry point with '--type era5land' + Test the hres_ic entry point """ with mock_sys_argv(num, start, _type): with patch("shutil.move", side_effect=new_shutil_move(num)): @@ -153,16 +157,36 @@ def test_hres_ic_era5land( num, output, expected_output ) - -def test_hres_eccb_era5land(): - """ - Test the hres_eccb entry point with '--type era5land' - """ - pass - - -def test_hres_eccb_barra(): +@pytest.mark.parametrize( + "num, start, _type", + [ + (4, "202305040000", "era5land"), + # (5, "202403050000", "barra"), + ], + ids=[ + "hres_eccb_era5land", + # "hres_eccb_barra", + ], +) +def test_hres_eccb( + new_shutil_move, + get_output_path, + get_expected_output_path, + get_error_msg, + mock_sys_argv, + num, + start, + _type, +): """ - Test the hres_eccb entry point with '--type barra' + Test the hres_eccb entry point """ - pass + with mock_sys_argv(num, start, _type): + with patch("shutil.move", side_effect=new_shutil_move(num)): + hres_ic.main() + output = get_output_path(num) + expected_output = get_expected_output_path(num) + # # Compare the output file with the expected output + assert filecmp.cmp(output, expected_output), get_error_msg( + num, output, expected_output + ) \ No newline at end of file From 3b035ebbbb0a7b9c76165b2e3f703dfca4aae108 Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Thu, 27 Feb 2025 12:11:19 +1100 Subject: [PATCH 12/23] Deleted old ibash ntegration tests --- integration/INTEGRATION_README.md | 28 ----- integration/integration_tests.sh | 182 ------------------------------ 2 files changed, 210 deletions(-) delete mode 100644 integration/INTEGRATION_README.md delete mode 100755 integration/integration_tests.sh diff --git a/integration/INTEGRATION_README.md b/integration/INTEGRATION_README.md deleted file mode 100644 index f568606..0000000 --- a/integration/INTEGRATION_README.md +++ /dev/null @@ -1,28 +0,0 @@ -# Integration tests for replace_landsurface - -`integration_tests.sh` is a basic binary compatibility test script for the replace_landsurface package. - -The script executes various tests in parallel, for all the entry points of the package, using different options to test multiple cases. - -All multiple tests are executed to completion, even if any of them fails. -When all the tests are completed, the script fails if any of the tested returned a non-zero exit code, and a message is printed with information about the failed test. - -The tests are designed to be run on Gadi within a Python environment where `replace_landsurface` is installed as a development package (for instructions refer to the Installation paragraph in the README). - -Usage: - regression_tests.sh [-h] [--keep] - -Options: - -h, --help Print this usage information and exit. - -k, --keep Force output data to be kept upon test completion. - The default behaviour is to keep the output data only for failed test sessions and delete it if all tests pass. - -All test data is located in `/g/data/vk83/testing/data/replace_landsurface/integration_tests`. - -### Tests performed - -- Test 1: hres_ic with `--type era5land` -- Test 2: hres_ic with `--type barra` -- Test 3: hres_ic with `--type astart` -- Test 4: hres_eccb with `--type era5land` (CURRENTLY NOT AVAILABLE) -- Test 5: hres_eccb with `--type barra` (CURRENTLY NOT AVAILABLE) \ No newline at end of file diff --git a/integration/integration_tests.sh b/integration/integration_tests.sh deleted file mode 100755 index da731b4..0000000 --- a/integration/integration_tests.sh +++ /dev/null @@ -1,182 +0,0 @@ -#!/bin/bash - -# Basic binary compatibility test script for replace_landsurface. -# See INTEGRATION_README.md for details on test usage, data and options. - -TEST_DATA_DIR=/g/data/vk83/testing/data/replace_landsurface/integration_tests -INPUT_DIR=$TEST_DATA_DIR/input_data -OUTPUT_DIR=$TEST_DATA_DIR/expected_outputs -DRIVING_DATA_DIR=$TEST_DATA_DIR/driving_data -CLEAN_OUTPUT=true - -declare -A pids # Associative array (similar to a dictionary in Python) - -# Create a temporary work directory -WORK_DIR=$(mktemp -d) -echo -e "Work directory: $WORK_DIR\n" -# Trap signals to clean up WORK directory depending on exit status -functrap() { - exit_code="$?" - if [ "$exit_code" -eq 0 ] && $CLEAN_OUTPUT; then # Job completed successfully and CLEAN_OUTPUT is true - rm -rf "$WORK_DIR" - echo "Work directory cleaned up." - elif [ "$exit_code" -eq 2 ]; then # Job was terminated preventively (did not fail) - kill -9 $(jobs -p) &> /dev/null # Kill all background jobs - rm -rf "$WORK_DIR" - echo "Script was terminated preventively." - echo "Work directory cleaned up. Background jobs killed." - fi -} -# Separate the cases when the script is interrupted, from the cases when it's not -trap "exit 2" SIGHUP SIGINT SIGQUIT SIGILL SIGABRT SIGTERM SIGSTOP -trap functrap EXIT - -start_time=$(date +%s) -#Set up the work directory as a copy of the test data directory -echo "Setting up work directory..." -cp -r $INPUT_DIR/* $WORK_DIR -cd $WORK_DIR -echo "Elapsed time for data copy: $(($(date +%s) - start_time)) seconds" - -function usage { - cat << EOF -Basic binary compatibility test script for 'replace_landsurface'. -Checks that the outputs of the 'replace_landsurface' package entry points correspond to the expected outputs. - -Usage: regression_tests.sh [-k/--keep] - -Options: --k, --keep Keep output data upon test completion. - If absent, output data will only be kept for failed tests. -EOF -} - -while getopts ":-:hk" opt; do - case ${opt} in - -) - case ${OPTARG} in - help) - usage - exit 0 - ;; - keep) - CLEAN_OUTPUT=false - ;; - *) - echo "Invalid option: \"--${OPTARG}\"." >&2 - usage - exit 1 - ;; - esac - ;; - h) - usage - exit 0 - ;; - k) - CLEAN_OUTPUT=true - ;; - \?) - echo "Invalid option: \"-${OPTARG}\"." >&2 - usage - exit 1 - ;; - esac -done - -# Check that no additional arguments were passed. -if [[ -n "${@:$OPTIND:1}" ]]; then - echo "Invalid positional argument: \"${@:$OPTIND:1}\"." >&2 - exit 1 -fi - -# # ----------------------------------------------------------------- -# # Run the tests -# # ----------------------------------------------------------------- -echo "Running tests..." - -export ROSE_DATA=$DRIVING_DATA_DIR -run_command() { - MASK=$WORK_DIR/$test_dir/mask - FILE=$WORK_DIR/$test_dir/file - HRES_IC=$WORK_DIR/$test_dir/hres_ic - eval "$entry_point --mask $MASK --file ${FILE}.tmp --start $START --hres_ic $HRES_IC --type $TYPE" -} -compare() { - test_num=${test_dir#test_} - cmp $WORK_DIR/$test_dir/file $OUTPUT_DIR/output_${test_num} - if [ $? -ne 0 ]; then - echo "Test $test_num failed." - exit 1 - fi -} -run_test() { - run_command - compare -} -# # ----------------------------------------------------------------- -# Test hres_ic -entry_point=hres_ic - -# Test 1: 'era5land' -# MASK=/scratch/tm70/cbe563/cylc-run/u-dg767/share/data/ancils/Lismore/d1000/qrparm.mask -# FILE=/g/data/tm70/cbe563/test_replace_land_surface/ERA5LAND/GAL9_astart -# HRES_IC=NOT_SET -test_dir=test_1 -TYPE=era5land -START=202202260000 - -echo "### Test 1: hres_ic, type 'era5land' ###" -run_test > /dev/null & pids[1]+=$! - -# # Test 2: 'barra' -# MASK=/scratch/tm70/cbe563/cylc-run/u-dg767.b/share/data/ancils/Lismore/d1100/qrparm.mask -# FILE=/g/data/tm70/cbe563/test_replace_land_surface/BARRAR2/GAL9_astart -# HRES_IC=NOT_SET -test_dir=test_2 -TYPE=barra -START=202008090000 - -echo "### Test 2: hres_ic, type 'barra' ###" -run_test > /dev/null & pids[2]+=$! - -# # Test 3: 'astart' -# MASK=/scratch/tm70/cbe563/cylc-run/u-dg767/share/data/ancils/Lismore/d0198/qrparm.mask -# FILE=/g/data/tm70/cbe563/test_replace_land_surface/ASTART/RAL3P2_astart -# HRES_IC=/scratch/tm70/cbe563/cylc-run/u-dg768.worked/share/cycle/20220226T0000Z/Lismore/d0198/RAL3P2/ics/RAL3P2_astart -test_dir=test_3 -TYPE=astart -START=202112310000 - -echo "### Test 3: hres_ic, type 'astart' ###" -run_test > /dev/null & pids[3]+=$! - -# Capture the exit status of each background processes dynamically -declare -A exit_statuses # Associative array (similar to a dictionary in Python) -for pid in "${pids[@]}"; do - # Find which test the pid corresponds to - for key in "${!pids[@]}"; do - if [[ ${pids[$key]} == $pid ]]; then - test_num=$key - break - fi - done - wait $pid # Waits for the next job in the list to finish (not necessarily in order because it's used within a loop) - exit_status=$? - exit_statuses[$test_num]=$exit_status -done - -# Exit with a status 1 if any of the tests failed -exit_value=0 -for ind in ${!exit_statuses[@]}; do - if [ ${exit_statuses[$ind]} -ne 0 ]; then - echo "Test $ind failed." - exit_value=1 - fi -done - -if [ $exit_value -eq 0 ]; then - echo "All tests passed." -else - exit 1 -fi \ No newline at end of file From a75033630d4fc07ed5f76b909ca610d4d35a5f66 Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Thu, 27 Feb 2025 12:57:33 +1100 Subject: [PATCH 13/23] Revert changes on src/replace_landsurface files --- .../replace_landsurface_with_BARRA2R_IC.py | 2 ++ .../replace_landsurface_with_ERA5land_IC.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py b/src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py index 4f64835..a420754 100755 --- a/src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py +++ b/src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py @@ -152,6 +152,7 @@ def get_BARRA_nc_data(ncfname, FIELDN, wanted_dt, NLAYERS, bounds): sys.exit(1) d.close() + return data.data @@ -201,6 +202,7 @@ def swap_land_barra(mask_fullpath, ec_cb_file_fullpath, ic_date): indir = os.path.join(BARRA_DIR, '1hr',BARRA_FIELDN, 'latest') barra_files = glob(os.path.join(indir, BARRA_FIELDN + '*' + yyyy + mm + '*nc')) barra_fname = indir + '/' + barra_files[0].split('/')[-1] + # Work out the grid bounds using the surface temperature file bounds = bounding_box(barra_fname, mask_fullpath.as_posix(), "land_binary_mask") diff --git a/src/replace_landsurface/replace_landsurface_with_ERA5land_IC.py b/src/replace_landsurface/replace_landsurface_with_ERA5land_IC.py index b744279..8d3bb92 100755 --- a/src/replace_landsurface/replace_landsurface_with_ERA5land_IC.py +++ b/src/replace_landsurface/replace_landsurface_with_ERA5land_IC.py @@ -215,6 +215,10 @@ def swap_land_era5land(mask_fullpath, ic_file_fullpath, ic_date): The file is replaced with a version of itself holding the higher-resolution data. """ + + # create name of file to be replaced + ic_file = ic_file_fullpath.parts[-1].replace('.tmp', '') + # create date/time useful information #print(ic_date) yyyy = ic_date[0:4] From fdbebc537cf235424822c80f7346ef6a9c5eafed Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Thu, 27 Feb 2025 17:04:37 +1100 Subject: [PATCH 14/23] Incorporated Tom's suggestions --- tests/integration/test_integration.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 1cba67c..68f91b4 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -4,6 +4,7 @@ import shutil import socket from unittest.mock import patch +from pathlib import Path import pytest # If not on Gadi, skip the tests because the test data is not available @@ -23,12 +24,12 @@ ############################################ ## === Integration tests setup === ## ############################################ -TEST_DATA_DIR = "/g/data/vk83/testing/data/replace_landsurface/integration_tests" -INPUT_DIR = os.path.join(TEST_DATA_DIR, "input_data") -OUTPUT_DIR = os.path.join(TEST_DATA_DIR, "expected_outputs") -DRIVING_DATA_DIR = os.path.join(TEST_DATA_DIR, "driving_data") +TEST_DATA_DIR = Path("/g/data/vk83/testing/data/replace_landsurface/integration_tests") +INPUT_DIR = TEST_DATA_DIR / "input_data" +OUTPUT_DIR = TEST_DATA_DIR / "expected_outputs" +DRIVING_DATA_DIR = TEST_DATA_DIR / "driving_data" # Set the ROSE_DATA environment variable to the driving data directory -os.environ["ROSE_DATA"] = DRIVING_DATA_DIR +os.environ["ROSE_DATA"] = str(DRIVING_DATA_DIR) from replace_landsurface import ( hres_ic, ) # importing here because we need to set the ROSE_DATA env variable before importing # noqa @@ -152,7 +153,7 @@ def test_hres_ic( hres_ic.main() output = get_output_path(num) expected_output = get_expected_output_path(num) - # # Compare the output file with the expected output + # Compare the output file with the expected output assert filecmp.cmp(output, expected_output), get_error_msg( num, output, expected_output ) @@ -186,7 +187,7 @@ def test_hres_eccb( hres_ic.main() output = get_output_path(num) expected_output = get_expected_output_path(num) - # # Compare the output file with the expected output + # Compare the output file with the expected output assert filecmp.cmp(output, expected_output), get_error_msg( num, output, expected_output ) \ No newline at end of file From 80f9957b1d680adae0541668050d0b2877de91ba Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Fri, 28 Feb 2025 14:49:56 +1100 Subject: [PATCH 15/23] Updated README --- README.md | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 42548af..8777daf 100644 --- a/README.md +++ b/README.md @@ -30,10 +30,19 @@ pip install --no-deps --no-build-isolation -e . ### Running the tests -#### Integration tests -To manually run the integration tests, from the `replace_landsurface` directory, you can: +The test suite currently includes only integration tests. + +To manually run the tests, from the `replace_landsurface` directory, you can: + 1. Activate your [micromamba/conda testing environment](#create-a-micromamba-conda-testing-environment) 2. Run the following command: ``` - bash integration/integration_tests.sh - ``` \ No newline at end of file + pytest -n 4 + ``` + +> [!TIP] +> The `-n 4` option is a [pytest-xdist](https://pytest-xdist.readthedocs.io/en/stable/) option to run the tests in parallel across 4 different workers. + +> [!IMPORTANT] +> Integration tests are designed to be run on `Gadi`. +> If you run tests on a local machine, the integration tests will be skipped. \ No newline at end of file From ce903760a9edeb0f97fed794f886f0dd9d6d569e Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Fri, 28 Feb 2025 14:59:27 +1100 Subject: [PATCH 16/23] Removed fixture to set env variable --- tests/integration/test_integration.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 68f91b4..d7adff4 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -98,13 +98,6 @@ def _get_expected_output_path(num): return _get_expected_output_path - -# Set the ROSE_DATA environment variable to the driving data directory in all tests -@pytest.fixture(autouse=True) -def mock_rose_data(monkeypatch): - monkeypatch.setenv("ROSE_DATA", DRIVING_DATA_DIR) - - @pytest.fixture(scope="module") def original_shutil_move(): return shutil.move From 0806a01e21db39651f97885f9459116d1d3c8161 Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Fri, 28 Feb 2025 15:05:02 +1100 Subject: [PATCH 17/23] Turned get_error_message fixture to a normal function --- tests/integration/test_integration.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index d7adff4..0dbe736 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -60,6 +60,10 @@ def get_test_args(num, start, _type): ] +def get_error_msg(num, output, expected_output): + return f"Test {num}: Test output '{output}' does not match the expected output '{expected_output}'!" + + @pytest.fixture def mock_sys_argv(): @contextlib.contextmanager @@ -70,14 +74,6 @@ def _mock_sys_argv(num, start, _type): return _mock_sys_argv -@pytest.fixture() -def get_error_msg(): - def _get_error_msg(num, output, expected_output): - return f"Test {num}: Test output '{output}' does not match the expected output '{expected_output}'!" - - return _get_error_msg - - @pytest.fixture(scope="module") def working_dir(tmp_path_factory): return tmp_path_factory.mktemp("replace_landsurface_integration_tests") @@ -132,7 +128,6 @@ def test_hres_ic( new_shutil_move, get_output_path, get_expected_output_path, - get_error_msg, mock_sys_argv, num, start, @@ -166,7 +161,6 @@ def test_hres_eccb( new_shutil_move, get_output_path, get_expected_output_path, - get_error_msg, mock_sys_argv, num, start, From abe59455e27fa70badb3a6da82ff58d7a3cc0bf5 Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Fri, 28 Feb 2025 15:08:06 +1100 Subject: [PATCH 18/23] Removed unused variable from fixture --- tests/integration/test_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 0dbe736..b8d786e 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -100,7 +100,7 @@ def original_shutil_move(): @pytest.fixture() -def new_shutil_move(original_shutil_move, get_output_path, get_expected_output_path): +def new_shutil_move(original_shutil_move, get_output_path): def _new_shutil_move(num): def _wrapper(src, dst, **kwargs): output_path = get_output_path(num) From 71800c3e37da437497c3a24b71152c1a5aa78e08 Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Fri, 28 Feb 2025 15:33:48 +1100 Subject: [PATCH 19/23] Turned all path concatenation to 'os.path.join' --- tests/integration/test_integration.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index b8d786e..14857ea 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -4,7 +4,6 @@ import shutil import socket from unittest.mock import patch -from pathlib import Path import pytest # If not on Gadi, skip the tests because the test data is not available @@ -24,15 +23,13 @@ ############################################ ## === Integration tests setup === ## ############################################ -TEST_DATA_DIR = Path("/g/data/vk83/testing/data/replace_landsurface/integration_tests") -INPUT_DIR = TEST_DATA_DIR / "input_data" -OUTPUT_DIR = TEST_DATA_DIR / "expected_outputs" -DRIVING_DATA_DIR = TEST_DATA_DIR / "driving_data" +TEST_DATA_DIR = "/g/data/vk83/testing/data/replace_landsurface/integration_tests" +INPUT_DIR = os.path.join(TEST_DATA_DIR, "input_data") +OUTPUT_DIR = os.path.join(TEST_DATA_DIR, "expected_outputs") +DRIVING_DATA_DIR = os.path.join(TEST_DATA_DIR, "driving_data") # Set the ROSE_DATA environment variable to the driving data directory os.environ["ROSE_DATA"] = str(DRIVING_DATA_DIR) -from replace_landsurface import ( - hres_ic, -) # importing here because we need to set the ROSE_DATA env variable before importing # noqa +from replace_landsurface import hres_ic # importing here because we need to set the ROSE_DATA env variable before importing # noqa ############################################ From deb4432e23dd9675ea8bd32c87c716c591acda47 Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Fri, 28 Feb 2025 19:53:39 +1100 Subject: [PATCH 20/23] Fixed hres_eccb test to call the right function --- tests/integration/test_integration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 14857ea..61050b3 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -29,7 +29,7 @@ DRIVING_DATA_DIR = os.path.join(TEST_DATA_DIR, "driving_data") # Set the ROSE_DATA environment variable to the driving data directory os.environ["ROSE_DATA"] = str(DRIVING_DATA_DIR) -from replace_landsurface import hres_ic # importing here because we need to set the ROSE_DATA env variable before importing # noqa +from replace_landsurface import hres_ic, hres_eccb # importing here because we need to set the ROSE_DATA env variable before importing # noqa ############################################ @@ -168,7 +168,7 @@ def test_hres_eccb( """ with mock_sys_argv(num, start, _type): with patch("shutil.move", side_effect=new_shutil_move(num)): - hres_ic.main() + hres_eccb.main() output = get_output_path(num) expected_output = get_expected_output_path(num) # Compare the output file with the expected output From 24359ce86b4a5307418e47bde24e02c39447f20b Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Sat, 1 Mar 2025 09:43:07 +1100 Subject: [PATCH 21/23] Removed hres_eccb script and renamed 'hres_ic' to 'replace_landsurface' to make it general. Also changed the entry points to only include replace_landsurface (old 'hres_ic'). --- pyproject.toml | 3 +- src/replace_landsurface/hres_eccb.py | 63 ------------------- .../{hres_ic.py => replace_landsurface.py} | 0 tests/integration/test_integration.py | 32 +++++----- 4 files changed, 16 insertions(+), 82 deletions(-) delete mode 100755 src/replace_landsurface/hres_eccb.py rename src/replace_landsurface/{hres_ic.py => replace_landsurface.py} (100%) diff --git a/pyproject.toml b/pyproject.toml index 4b36a01..0449a71 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,8 +26,7 @@ dependencies = [ Repository = "https://github.com/ACCESS-NRI/replace_landsurface" [project.scripts] -hres_eccb = "replace_landsurface.hres_eccb:main" -hres_ic = "replace_landsurface.hres_ic:main" +replace_landsurface = "replace_landsurface.replace_landsurface:main" [build-system] build-backend = "setuptools.build_meta" diff --git a/src/replace_landsurface/hres_eccb.py b/src/replace_landsurface/hres_eccb.py deleted file mode 100755 index b792da1..0000000 --- a/src/replace_landsurface/hres_eccb.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2024 ACCESS-NRI (https://www.access-nri.org.au/) -# See the top-level COPYRIGHT.txt file for details. -# -# SPDX-License-Identifier: Apache-2.0 -# -# Created by: Chermelle Engel - -""" -Replace the land/surface fields in the ec_cb000 file with higher-resolution -era5-land or BARRA2-R data (if requested). -""" - -import argparse -import shutil -from pathlib import Path - -import pandas - -from replace_landsurface import replace_landsurface_with_BARRA2R_IC, replace_landsurface_with_ERA5land_IC - -def main(): - """ - The main function that creates a worker pool and generates single GRIB files - for requested date/times in parallel. - - Parameters - ---------- - None. The arguments are given via the command-line - - Returns - ------- - None. The ec_cb000 file is updated and overwritten - """ - - # Parse the command-line arguments - parser = argparse.ArgumentParser() - parser.add_argument('--mask', required=True, type=Path) - parser.add_argument('--file', required=True, type=Path) - parser.add_argument('--start', required=True, type=pandas.to_datetime) - parser.add_argument('--type', default="era5land") - parser.add_argument('--hres_ic', type=Path) - args = parser.parse_args() - print(args) - - # Convert the date/time to a formatted string - t = args.start.strftime("%Y%m%dT%H%MZ") - print(args.mask, args.file, t) - - # If necessary replace ERA5 land/surface fields with higher-resolution options - if "era5land" in args.type: - replace_landsurface_with_ERA5land_IC.swap_land_era5land(args.mask, args.file, t) - shutil.move(args.file.as_posix(), args.file.as_posix().replace('.tmp', '')) - elif "barra" in args.type: - replace_landsurface_with_BARRA2R_IC.swap_land_barra(args.mask, args.file, t) - shutil.move(args.file.as_posix(), args.file.as_posix().replace('.tmp', '')) - elif "astart" in args.type: - print("Fields not swapped out for ECCB files when using start dump as replacement option.") - else: - print("No need to swap out IC") - -if __name__ == '__main__': - main() - diff --git a/src/replace_landsurface/hres_ic.py b/src/replace_landsurface/replace_landsurface.py similarity index 100% rename from src/replace_landsurface/hres_ic.py rename to src/replace_landsurface/replace_landsurface.py diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 61050b3..687987b 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -29,7 +29,7 @@ DRIVING_DATA_DIR = os.path.join(TEST_DATA_DIR, "driving_data") # Set the ROSE_DATA environment variable to the driving data directory os.environ["ROSE_DATA"] = str(DRIVING_DATA_DIR) -from replace_landsurface import hres_ic, hres_eccb # importing here because we need to set the ROSE_DATA env variable before importing # noqa +from replace_landsurface import replace_landsurface # importing here because we need to set the ROSE_DATA env variable before importing # noqa ############################################ @@ -37,23 +37,23 @@ ############################################ def get_test_args(num, start, _type): test_dir = f"test_{num}" - _hres_ic = ( + hres_ic = ( os.path.join(INPUT_DIR, test_dir, "hres_ic") if _type == "astart" else "NOT_USED" ) return [ "script_name", - "--mask", - os.path.join(INPUT_DIR, test_dir, "mask"), "--file", os.path.join(INPUT_DIR, test_dir, "file" + ".tmp"), + "--mask", + os.path.join(INPUT_DIR, test_dir, "mask"), "--start", start, "--type", _type, "--hres_ic", - _hres_ic, + hres_ic, ] @@ -116,12 +116,12 @@ def _wrapper(src, dst, **kwargs): (3, "202112310000", "astart"), ], ids=[ - "hres_ic_era5land", - "hres_ic_barra", - "hres_ic_astart", + "replace_landsurface_era5land", + "replace_landsurface_barra", + "replace_landsurface_astart", ], ) -def test_hres_ic( +def test_replace_landsurface( new_shutil_move, get_output_path, get_expected_output_path, @@ -131,11 +131,11 @@ def test_hres_ic( _type, ): """ - Test the hres_ic entry point + Test the replace_landsurface entry point """ with mock_sys_argv(num, start, _type): with patch("shutil.move", side_effect=new_shutil_move(num)): - hres_ic.main() + replace_landsurface.main() output = get_output_path(num) expected_output = get_expected_output_path(num) # Compare the output file with the expected output @@ -147,14 +147,12 @@ def test_hres_ic( "num, start, _type", [ (4, "202305040000", "era5land"), - # (5, "202403050000", "barra"), ], ids=[ - "hres_eccb_era5land", - # "hres_eccb_barra", + "replace_landsurface_era5land_2", ], ) -def test_hres_eccb( +def test_replace_landsurface_2( new_shutil_move, get_output_path, get_expected_output_path, @@ -164,11 +162,11 @@ def test_hres_eccb( _type, ): """ - Test the hres_eccb entry point + Test the replace_landsurface entry point """ with mock_sys_argv(num, start, _type): with patch("shutil.move", side_effect=new_shutil_move(num)): - hres_eccb.main() + replace_landsurface.main() output = get_output_path(num) expected_output = get_expected_output_path(num) # Compare the output file with the expected output From 8eec4fe384641d480d66ab3051d14b89f4c46a52 Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Sat, 1 Mar 2025 10:12:28 +1100 Subject: [PATCH 22/23] Embedded old 'test_hres_eccb' test (test n. 4) within the 'test_replace_landsurface' function --- tests/integration/test_integration.py | 33 ++------------------------- 1 file changed, 2 insertions(+), 31 deletions(-) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 687987b..db238e5 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -114,11 +114,13 @@ def _wrapper(src, dst, **kwargs): (1, "202202260000", "era5land"), (2, "202008090000", "barra"), (3, "202112310000", "astart"), + (4, "202305040000", "era5land"), ], ids=[ "replace_landsurface_era5land", "replace_landsurface_barra", "replace_landsurface_astart", + "replace_landsurface_era5land_2", ], ) def test_replace_landsurface( @@ -129,37 +131,6 @@ def test_replace_landsurface( num, start, _type, -): - """ - Test the replace_landsurface entry point - """ - with mock_sys_argv(num, start, _type): - with patch("shutil.move", side_effect=new_shutil_move(num)): - replace_landsurface.main() - output = get_output_path(num) - expected_output = get_expected_output_path(num) - # Compare the output file with the expected output - assert filecmp.cmp(output, expected_output), get_error_msg( - num, output, expected_output - ) - -@pytest.mark.parametrize( - "num, start, _type", - [ - (4, "202305040000", "era5land"), - ], - ids=[ - "replace_landsurface_era5land_2", - ], -) -def test_replace_landsurface_2( - new_shutil_move, - get_output_path, - get_expected_output_path, - mock_sys_argv, - num, - start, - _type, ): """ Test the replace_landsurface entry point From 3b0df76b3f4b5b2a1e3a3396deb2b8b93e38e0c8 Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Sat, 1 Mar 2025 10:21:04 +1100 Subject: [PATCH 23/23] Updated docstrings --- src/replace_landsurface/replace_landsurface.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/replace_landsurface/replace_landsurface.py b/src/replace_landsurface/replace_landsurface.py index d29ce6e..5604d9b 100755 --- a/src/replace_landsurface/replace_landsurface.py +++ b/src/replace_landsurface/replace_landsurface.py @@ -6,8 +6,7 @@ # Created by: Chermelle Engel """ -Replace the land/surface fields in the astart file with higher-resolution -era5-land or BARRA2-R data (if requested). +Replace the land-surface fields in the astart file with higher-resolution data """ import argparse @@ -25,16 +24,17 @@ def main(): """ - The main function that creates a worker pool and generates single GRIB files - for requested date/times in parallel. + Calls the command line argument parser and process the arguments using the right function. Parameters ---------- - None. The arguments are given via the command-line + None + The arguments are given via the command-line Returns ------- - None. The astart file is updated and overwritten + None + An output file is written """ # Parse the command-line arguments