From 09b75ae867ca76b3ea95bdded4504f105417d4a2 Mon Sep 17 00:00:00 2001 From: Tom Russell Date: Fri, 5 Apr 2024 10:44:35 +0100 Subject: [PATCH 1/5] Update STORM fixed return period URLs --- config/hazard_resource_locations/storm_fixed_CMCC-CM2-VHR4.txt | 2 +- config/hazard_resource_locations/storm_fixed_CNRM-CM6-1-HR.txt | 2 +- config/hazard_resource_locations/storm_fixed_EC-Earth3P-HR.txt | 2 +- .../hazard_resource_locations/storm_fixed_HadGEM3-GC31-HM.txt | 2 +- config/hazard_resource_locations/storm_fixed_constant.txt | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/config/hazard_resource_locations/storm_fixed_CMCC-CM2-VHR4.txt b/config/hazard_resource_locations/storm_fixed_CMCC-CM2-VHR4.txt index ea92a0a9..d1b3aa3e 100644 --- a/config/hazard_resource_locations/storm_fixed_CMCC-CM2-VHR4.txt +++ b/config/hazard_resource_locations/storm_fixed_CMCC-CM2-VHR4.txt @@ -1 +1 @@ -https://data.4tu.nl/ndownloader/files/35110321 +https://data.4tu.nl/file/504c838e-2bd8-4d61-85a1-d495bdc560c3/2a3759e8-f4ad-4190-abb3-434ad5c4679b diff --git a/config/hazard_resource_locations/storm_fixed_CNRM-CM6-1-HR.txt b/config/hazard_resource_locations/storm_fixed_CNRM-CM6-1-HR.txt index 24289f9c..60cde7ea 100644 --- a/config/hazard_resource_locations/storm_fixed_CNRM-CM6-1-HR.txt +++ b/config/hazard_resource_locations/storm_fixed_CNRM-CM6-1-HR.txt @@ -1 +1 @@ -https://data.4tu.nl/ndownloader/files/35110318 +https://data.4tu.nl/file/504c838e-2bd8-4d61-85a1-d495bdc560c3/c460a0c8-f918-4ede-a734-75e77e99b102 diff --git a/config/hazard_resource_locations/storm_fixed_EC-Earth3P-HR.txt b/config/hazard_resource_locations/storm_fixed_EC-Earth3P-HR.txt index a290c2e3..5ea8b0df 100644 --- a/config/hazard_resource_locations/storm_fixed_EC-Earth3P-HR.txt +++ b/config/hazard_resource_locations/storm_fixed_EC-Earth3P-HR.txt @@ -1 +1 @@ -https://data.4tu.nl/ndownloader/files/35110312 +https://data.4tu.nl/file/504c838e-2bd8-4d61-85a1-d495bdc560c3/304d1441-bd71-47c7-8231-b20253c1cc2a diff --git a/config/hazard_resource_locations/storm_fixed_HadGEM3-GC31-HM.txt b/config/hazard_resource_locations/storm_fixed_HadGEM3-GC31-HM.txt index 2a5bc7f4..130c6b80 100644 --- a/config/hazard_resource_locations/storm_fixed_HadGEM3-GC31-HM.txt +++ b/config/hazard_resource_locations/storm_fixed_HadGEM3-GC31-HM.txt @@ -1 +1 @@ -https://data.4tu.nl/ndownloader/files/35110309 +https://data.4tu.nl/file/504c838e-2bd8-4d61-85a1-d495bdc560c3/856f9530-56d7-489e-8005-18ae36db4804 diff --git a/config/hazard_resource_locations/storm_fixed_constant.txt b/config/hazard_resource_locations/storm_fixed_constant.txt index 211db172..a9fc8c92 100644 --- a/config/hazard_resource_locations/storm_fixed_constant.txt +++ b/config/hazard_resource_locations/storm_fixed_constant.txt @@ -1 +1 @@ -https://data.4tu.nl/ndownloader/files/35423735 +https://data.4tu.nl/file/0ea98bdd-5772-4da8-ae97-99735e891aff/0b98c6f5-c7af-45bb-bb6e-dac53e1b8d55 From ac9c2a8b5941f5634694481793140d64aab9b79d Mon Sep 17 00:00:00 2001 From: Tom Russell Date: Fri, 5 Apr 2024 10:45:41 +0100 Subject: [PATCH 2/5] Rename extracted STORM files to match expected case --- workflow/tropical-cyclone/STORM.smk | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/workflow/tropical-cyclone/STORM.smk b/workflow/tropical-cyclone/STORM.smk index 71a2e368..ee3793e5 100644 --- a/workflow/tropical-cyclone/STORM.smk +++ b/workflow/tropical-cyclone/STORM.smk @@ -179,12 +179,19 @@ rule extract_storm_fixed_future: STORM_MODEL_UPPER = lambda wildcards: wildcards.STORM_MODEL_FUTURE.upper() shell: """ - # first try STORM_MODEL_FUTURE, otherwise fall back to trying the uppercase version - unzip -o {input} STORM_FIXED_RETURN_PERIODS_{wildcards.STORM_MODEL_FUTURE}_{wildcards.STORM_BASIN}_{wildcards.STORM_RP}_YR_RP.tif \ - -d {wildcards.OUTPUT_DIR}/input/STORM/fixed/{wildcards.STORM_MODEL_FUTURE}/{wildcards.STORM_BASIN}/ \ - || \ - unzip -o {input} STORM_FIXED_RETURN_PERIODS_{params.STORM_MODEL_UPPER}_{wildcards.STORM_BASIN}_{wildcards.STORM_RP}_YR_RP.tif \ - -d {wildcards.OUTPUT_DIR}/input/STORM/fixed/{wildcards.STORM_MODEL_FUTURE}/{wildcards.STORM_BASIN}/ \ + unpack_dir="{wildcards.OUTPUT_DIR}/input/STORM/fixed/{wildcards.STORM_MODEL_FUTURE}/{wildcards.STORM_BASIN}/" + uppercase_filename="STORM_FIXED_RETURN_PERIODS_{params.STORM_MODEL_UPPER}_{wildcards.STORM_BASIN}_{wildcards.STORM_RP}_YR_RP.tif" + desired_filename="STORM_FIXED_RETURN_PERIODS_{wildcards.STORM_MODEL_FUTURE}_{wildcards.STORM_BASIN}_{wildcards.STORM_RP}_YR_RP.tif" + + # try extracting filename with STORM_MODEL_FUTURE, fall back to STORM_MODEL_UPPER + unzip -o {input} $desired_filename -d $unpack_dir || unzip -o {input} $uppercase_filename -d $unpack_dir + + # if we unpacked the uppercase version and its different to STORM_MODEL_FUTURE, rename it + pushd $unpack_dir + if [ -f $uppercase_filename ] && [ $uppercase_filename != $desired_filename ]; then + mv $uppercase_filename $desired_filename + fi + popd """ From 5bb023f667ec8655d38413232f75fd29587908d8 Mon Sep 17 00:00:00 2001 From: Tom Russell Date: Fri, 5 Apr 2024 10:48:59 +0100 Subject: [PATCH 3/5] Wrap STORM files to extent aligned with source pixels This was half a pixel (0.05 degrees) out, as the source data is not aligned with integer degrees at the upper-left corner of cells. --- workflow/tropical-cyclone/STORM.smk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/tropical-cyclone/STORM.smk b/workflow/tropical-cyclone/STORM.smk index ee3793e5..a9ca0c03 100644 --- a/workflow/tropical-cyclone/STORM.smk +++ b/workflow/tropical-cyclone/STORM.smk @@ -202,7 +202,7 @@ rule wrap_storm_fixed: temp("{OUTPUT_DIR}/input/STORM/fixed/{STORM_MODEL}/{STORM_BASIN}/STORM_FIXED_RETURN_PERIODS_{STORM_MODEL}_{STORM_BASIN}_{STORM_RP}_YR_RP.wrapped.tif") shell: """ - gdalwarp -te -180 -60 180 60 {input} {output} + gdalwarp -te -179.85 -60.15 180.15 59.95 -co COMPRESS=LZW -co PREDICTOR=2 -co TILED=YES {input} {output} """ From d09dc176835403ab5fba5c14ef36d464f39f9ef1 Mon Sep 17 00:00:00 2001 From: Tom Russell Date: Fri, 5 Apr 2024 10:49:40 +0100 Subject: [PATCH 4/5] Compress STORM return period map files --- workflow/tropical-cyclone/STORM.smk | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/workflow/tropical-cyclone/STORM.smk b/workflow/tropical-cyclone/STORM.smk index a9ca0c03..cbbc446e 100644 --- a/workflow/tropical-cyclone/STORM.smk +++ b/workflow/tropical-cyclone/STORM.smk @@ -230,5 +230,8 @@ rule mosaic_storm_fixed: -F {input.basin_tif_wp} \ --outfile={output} \ --calc="numpy.max((A,B,C,D,E,F),axis=0)" \ - --NoDataValue=0 + --NoDataValue=0 \ + --creation-option="COMPRESS=LZW" \ + --creation-option="PREDICTOR=2" \ + --creation-option="TILED=YES" """ From 07c58a5ae55693385bb57e28a1788440027cf5b9 Mon Sep 17 00:00:00 2001 From: Tom Russell Date: Fri, 5 Apr 2024 10:50:14 +0100 Subject: [PATCH 5/5] Add convenience rule for all global STORM return period files --- workflow/tropical-cyclone/STORM.smk | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/workflow/tropical-cyclone/STORM.smk b/workflow/tropical-cyclone/STORM.smk index cbbc446e..07df0b9b 100644 --- a/workflow/tropical-cyclone/STORM.smk +++ b/workflow/tropical-cyclone/STORM.smk @@ -235,3 +235,23 @@ rule mosaic_storm_fixed: --creation-option="PREDICTOR=2" \ --creation-option="TILED=YES" """ + +rule mosaic_storm_fixed_all: + input: + tiffs=expand( + "{{OUTPUT_DIR}}/input/STORM/fixed/{STORM_MODEL}/STORM_FIXED_RETURN_PERIODS_{STORM_MODEL}_{STORM_RP}_YR_RP.tif", + STORM_MODEL=[ + "constant", + "CMCC-CM2-VHR4", + "CNRM-CM6-1-HR", + "EC-Earth3P-HR", + "HadGEM3-GC31-HM", + ], + STORM_RP=( + list(range(10, 100, 10)) + + list(range(100, 1000, 100)) + + list(range(1000, 10001, 1000)) + ), + ) + output: + touch("{OUTPUT_DIR}/input/STORM/fixed/mosaic.done")