From 6891f96e0f9a9f12a3c70c6210123e4163b7c7d9 Mon Sep 17 00:00:00 2001 From: Rose Dickinson Date: Fri, 15 Jan 2021 14:53:47 +0000 Subject: [PATCH 1/2] fix: Various small problems with the images and adding documentation --- DAFNI-wrappers/Readme.md | 14 ++++++++++++ DAFNI-wrappers/base-image/README.md | 20 +++++++++++++++-- DAFNI-wrappers/decide_step/README.md | 9 +++++++- .../decide_step/dockerFiles/extract_data.py | 2 +- .../decide_step/model_definition.yaml | 20 +++-------------- DAFNI-wrappers/energy_demand/README.md | 3 ++- .../energy_demand/dockerFiles/extract_data.py | 2 +- DAFNI-wrappers/energy_supply/README.md | 22 ++++++++++++++++++- .../model/dockerFiles/extract_data.py | 2 +- DAFNI-wrappers/et_module/README.md | 3 ++- DAFNI-wrappers/transforms/README.md | 4 +++- .../transforms/dockerFiles/extract_data.py | 2 +- .../transforms/model_definition.yaml | 6 ++--- DAFNI-wrappers/transport/README.md | 3 ++- 14 files changed, 80 insertions(+), 32 deletions(-) create mode 100644 DAFNI-wrappers/Readme.md diff --git a/DAFNI-wrappers/Readme.md b/DAFNI-wrappers/Readme.md new file mode 100644 index 00000000..2a98f6a5 --- /dev/null +++ b/DAFNI-wrappers/Readme.md @@ -0,0 +1,14 @@ +# DAFNI Wrappers + +These are the Docker images that wrap the NISMOD2 models to allow them to be run on +DAFNI. + +Each of the images has their own readme file. Here are just some things to keep in mind +when changing the images: + +- The extract_data.py scripts currently do reference the data packs by file name + directly so if you update the data packs you will also need to update the + extract_data.py scripts to make sure they are correct too. +- The decide_step and transforms wrappers define their own run_nismod.py script that + overwrites the one in the base image, this means any changes to the script in the base + image won't apply to those wrappers. diff --git a/DAFNI-wrappers/base-image/README.md b/DAFNI-wrappers/base-image/README.md index 586a2c27..7b1d93a6 100644 --- a/DAFNI-wrappers/base-image/README.md +++ b/DAFNI-wrappers/base-image/README.md @@ -1,3 +1,19 @@ -# transport +# Base Image -Base image for all nismod models \ No newline at end of file +This is the base Docker image for all the other DAFNI-wrappers for NISMOD2 models. This +image contains all of the packages and scripts that are common between the models. + +The settings.py file sets some commonly used paths and also grabs the parameters from +the environment variables that DAFNI sets on the container when the Model is run. This +is how users can specify the parameter values via Workflows. + +utils.py just contains some useful functions that are used by the wrapper scripts. + +run_nismod.py contains the code that actually runs the NISMOD2 model based on the +parameters the user provided. + +job_processing_wrapper.py is the script DAFNI calls to run the models, it simply calls +run_nismod.py and then copies the results of the model into the proper output location. + +You shouldn't need to build this image locally in order to build the other models as +they all use the image in the github registry as their base. diff --git a/DAFNI-wrappers/decide_step/README.md b/DAFNI-wrappers/decide_step/README.md index fae9dedf..85ab0361 100644 --- a/DAFNI-wrappers/decide_step/README.md +++ b/DAFNI-wrappers/decide_step/README.md @@ -1,3 +1,10 @@ # Decide Step -NIMS wrapper for NISMOD decision steps \ No newline at end of file +This is the DAFNI wrapper for the NISMOD decision step. The reason this step has been +extracted out is so that each of the other NISMOD models can only require the datasets +they directly use. If this step wasn't extracted out then every model would need a copy +of every dataset in order to be properly used in a SOS Workflow. + +This is a very simple image, it just extracts all the datasets into their proper +locations then runs the decision step. In order to create a SOS Workflow this should be +the first step of the Workflow. diff --git a/DAFNI-wrappers/decide_step/dockerFiles/extract_data.py b/DAFNI-wrappers/decide_step/dockerFiles/extract_data.py index 65b8571f..9e6cd815 100644 --- a/DAFNI-wrappers/decide_step/dockerFiles/extract_data.py +++ b/DAFNI-wrappers/decide_step/dockerFiles/extract_data.py @@ -36,7 +36,7 @@ def extract(): "dest": str(NISMOD_SCENARIOS_PATH.joinpath("ev_transport_trips/")), }, { - "src": "/data/energy_supply/energy_supply_data_v0.9.10.zip", + "src": "/data/energy_supply/energy_supply_data_v0.9.12.zip", "dest": str(NISMOD_DATA_PATH.joinpath("energy_supply/")), }, { diff --git a/DAFNI-wrappers/decide_step/model_definition.yaml b/DAFNI-wrappers/decide_step/model_definition.yaml index 8943ba7c..bd7cf8fc 100644 --- a/DAFNI-wrappers/decide_step/model_definition.yaml +++ b/DAFNI-wrappers/decide_step/model_definition.yaml @@ -1,7 +1,7 @@ kind: Model apiVersion: v1alpha4 metadata: - displayName: NISMOD - Decision Module (model v2.4.0 build 1) + displayName: NISMOD - Decision Module (model v2.4.0 build 5) name: nismod-decide summary: National Infrastructure Systems Model type: systems of systems @@ -15,29 +15,15 @@ metadata: spec: inputs: env: - - name: sector_model - title: NISMOD2 Sector Model - desc: Pick one of the NISMOD2 Sector Models to run the decide step for - type: string - name: model_to_run title: Model Run desc: The name of the model run that is being executed type: string - default: energy_supply_constrained - - name: part_of_sos_model - title: Part of SOS Model - desc: "Is this model being run as part of a NISMOD2 Systems of Systems Model?" - type: boolean - default: false - name: use_generated_scenario title: Use generated scenario desc: "Is this run using a new generated scenario file?" type: boolean default: false - - name: timestep - title: Timestep - desc: "Timestep to run the Model at if this is part of an SoS Model" - type: integer dataslots: - name: Scenario Data description: > @@ -93,8 +79,8 @@ spec: vehicle trip starts and consumption scenarios (test outputs from the Transport model used to test running the et_module standalone). default: - - uid: dbfe6814-3fd3-4e94-8c30-ccfec60c9989 - versionUid: df234857-a622-4b83-bec1-c72b1c7ed8b6 + - uid: ed671698-069c-40f4-8645-bf08e45634fb + versionUid: 66382de9-bef7-45ab-b729-dbfc1d01007f path: et_module/ required: true - name: LAD boundaries diff --git a/DAFNI-wrappers/energy_demand/README.md b/DAFNI-wrappers/energy_demand/README.md index b462d731..701d9002 100644 --- a/DAFNI-wrappers/energy_demand/README.md +++ b/DAFNI-wrappers/energy_demand/README.md @@ -1,3 +1,4 @@ # energy_demand -NIMS wrapper for NISMOD energy_demand model \ No newline at end of file +The DAFNI wrapper for NISMOD2's energy_demand model. Again this is a fairly simple +wrapper, that extracts the energy demand datasets and then runs the Model. diff --git a/DAFNI-wrappers/energy_demand/dockerFiles/extract_data.py b/DAFNI-wrappers/energy_demand/dockerFiles/extract_data.py index 458ad9cf..b8b0a86d 100644 --- a/DAFNI-wrappers/energy_demand/dockerFiles/extract_data.py +++ b/DAFNI-wrappers/energy_demand/dockerFiles/extract_data.py @@ -35,7 +35,7 @@ def extract(): "dest": str(NISMOD_SCENARIOS_PATH.joinpath("ev_transport_trips/")), }, # { - # "src": "/data/energy_supply/energy_supply_data_v0.9.10.zip", + # "src": "/data/energy_supply/energy_supply_data_v0.9.12.zip", # "dest": str(NISMOD_DATA_PATH.joinpath("energy_supply/")), # }, { diff --git a/DAFNI-wrappers/energy_supply/README.md b/DAFNI-wrappers/energy_supply/README.md index a704d65e..39430da3 100644 --- a/DAFNI-wrappers/energy_supply/README.md +++ b/DAFNI-wrappers/energy_supply/README.md @@ -1,3 +1,23 @@ # energy_supply -NIMS wrapper for NISMOD energy_supply model \ No newline at end of file +The DAFNI wrapper for NISMOD2's energy_supply model. This model is quite complex, the +current setup on DAFNI has the postgres database running in a sidecar (a container that +runs along side the main model container but that is linked by a local network). The +image for this sidecar is in the db folder, unfortunately DAFNI doesn't currently have a +good process to allow users to customise and specify sidecars themselves so this image +is provided purely for completeness and to allow you to run the model locally. + +I have separated out the model image into a new model_base image (which adds packages to +the generic base image) and the model image which actually containers the wrapper +scripts. I have done this because the install for FICO and the other packages in the +model_base image takes a long time and also because these are unlikely to ever change. +The model image itself is quite simple, it contains the wrapper scripts and a built +version of the energy supply model. The model and the datasets will be extracted to the +right places and the model will then be run. + +I have provided a docker-compose file that should allow you to build and run the energy +supply locally with minimal hassle. + +One thing that hasn't been included in the repository is the license file for FICO that +this wrapper needs in order to run the energy supply model. You will unfortunately have +to provide this yourself as FICO is commercial software. diff --git a/DAFNI-wrappers/energy_supply/model/dockerFiles/extract_data.py b/DAFNI-wrappers/energy_supply/model/dockerFiles/extract_data.py index 855615ff..94ab7ba2 100644 --- a/DAFNI-wrappers/energy_supply/model/dockerFiles/extract_data.py +++ b/DAFNI-wrappers/energy_supply/model/dockerFiles/extract_data.py @@ -108,7 +108,7 @@ def extract(): "dest": str(NISMOD_SCENARIOS_PATH.joinpath("ev_transport_trips/")), }, { - "src": "/data/energy_supply/energy_supply_data_v0.9.10.zip", + "src": "/data/energy_supply/energy_supply_data_v0.9.12.zip", "dest": str(NISMOD_DATA_PATH.joinpath("energy_supply/")), }, # { diff --git a/DAFNI-wrappers/et_module/README.md b/DAFNI-wrappers/et_module/README.md index bdbea5de..d03ffaa4 100644 --- a/DAFNI-wrappers/et_module/README.md +++ b/DAFNI-wrappers/et_module/README.md @@ -1,3 +1,4 @@ # et_module -Docker image for the et_module in nismod2 \ No newline at end of file +The DAFNI wrapper for NISMOD2's et_module model. Again this is a fairly simple +wrapper, that extracts the et module datasets and then runs the model. diff --git a/DAFNI-wrappers/transforms/README.md b/DAFNI-wrappers/transforms/README.md index d7703332..d93446b3 100644 --- a/DAFNI-wrappers/transforms/README.md +++ b/DAFNI-wrappers/transforms/README.md @@ -1,3 +1,5 @@ # nismod-transforms -Model that runs the transformation / conversion steps for nismod2. \ No newline at end of file +This DAFNI wrapper allows a user to run one (or more) of the various adaptors / +transform operations that are required to convert data to the appropriate forms when +going from model to model in a SOS Workflow. diff --git a/DAFNI-wrappers/transforms/dockerFiles/extract_data.py b/DAFNI-wrappers/transforms/dockerFiles/extract_data.py index a4828fc8..09febe4f 100644 --- a/DAFNI-wrappers/transforms/dockerFiles/extract_data.py +++ b/DAFNI-wrappers/transforms/dockerFiles/extract_data.py @@ -48,7 +48,7 @@ def extract(): "dest": str(NISMOD_SCENARIOS_PATH.joinpath("ev_transport_trips/")), }, { - "src": "/data/energy_supply/energy_supply_data_v0.9.10.zip", + "src": "/data/energy_supply/energy_supply_data_v0.9.12.zip", "dest": str(NISMOD_DATA_PATH.joinpath("energy_supply/")), }, { diff --git a/DAFNI-wrappers/transforms/model_definition.yaml b/DAFNI-wrappers/transforms/model_definition.yaml index 8ffbe115..dd86b386 100644 --- a/DAFNI-wrappers/transforms/model_definition.yaml +++ b/DAFNI-wrappers/transforms/model_definition.yaml @@ -1,7 +1,7 @@ kind: Model apiVersion: v1alpha4 metadata: - displayName: NISMOD - Adaptors (model v2.4.0 build 1) + displayName: NISMOD - Adaptors (model v2.4.0 build 3) name: nismod-transforms summary: National Infrastructure Systems Model type: systems of systems @@ -93,8 +93,8 @@ spec: vehicle trip starts and consumption scenarios (test outputs from the Transport model used to test running the et_module standalone). default: - - uid: dbfe6814-3fd3-4e94-8c30-ccfec60c9989 - versionUid: df234857-a622-4b83-bec1-c72b1c7ed8b6 + - uid: ed671698-069c-40f4-8645-bf08e45634fb + versionUid: 66382de9-bef7-45ab-b729-dbfc1d01007f path: et_module/ required: true - name: LAD boundaries diff --git a/DAFNI-wrappers/transport/README.md b/DAFNI-wrappers/transport/README.md index aa7f00fd..ca994df6 100644 --- a/DAFNI-wrappers/transport/README.md +++ b/DAFNI-wrappers/transport/README.md @@ -1,3 +1,4 @@ # transport -Docker image for the NISMOD2 transport model \ No newline at end of file +The DAFNI wrapper for NISMOD2's transport model. Again this is a fairly simple wrapper, +that extracts the transport datasets and then runs the model. From 940b54144d58972af4b5dde9ce894ce68df83383 Mon Sep 17 00:00:00 2001 From: Rose Dickinson Date: Thu, 28 Jan 2021 16:03:33 +0000 Subject: [PATCH 2/2] fix: UUIDs for datasets --- .../energy_demand/model_definition.yaml | 42 +++++++++++++------ .../et_module/model_definition.yaml | 4 +- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/DAFNI-wrappers/energy_demand/model_definition.yaml b/DAFNI-wrappers/energy_demand/model_definition.yaml index 0f69fa74..132aae94 100644 --- a/DAFNI-wrappers/energy_demand/model_definition.yaml +++ b/DAFNI-wrappers/energy_demand/model_definition.yaml @@ -1,10 +1,10 @@ kind: Model -apiVersion: v1alpha3 +apiVersion: v1alpha4 metadata: # You will probably want to update this metadata before uploading to DAFNI - displayName: NISMOD2 - Energy Demand Model + displayName: NISMOD2 - High Resolution Energy Demand Model (HIRE, v0.9) name: nismod-energy-demand - summary: National Infrastructure Systems Model + summary: National Infrastructure Systems Model (preliminary upload No.9) type: systems of systems description: "Energy Demand model that forms part of NISMOD2" spec: @@ -40,17 +40,33 @@ spec: title: Timestep desc: "Simulation year (used if run as part of a NISMOD Systems of Systems Model)" type: integer - datasets: + dataslots: # All the below UUIDs are for DAFNI's staging site, they will need updating before uploading to DAFNI production - - uid: 1c25acf9-8f35-413f-8995-93ac4876eeac - versionUid: 22bf0529-7f21-43d8-bb11-ce84dbffa123 + - name: Scenario Data + description: > + NISMOD scenario data pack. Should contain socio-economic scenarios + (population, GVA) and energy (fuel) price scenarios. + default: + - uid: 1237868f-52ea-416c-9087-eef9dbde429d + versionUid: 5afbfaa2-a351-4a04-9b79-efa6a194aa60 path: scenarios/ - - uid: 3a27a317-1ace-48b6-8515-4889cbc3ed64 - versionUid: fb325d6a-5ada-4844-9619-625617b59e4a + required: true + - name: Energy Demand Model Data + description: > + NISMOD energy demand data pack. Should contain full data setup for the + energy demand model. + default: + - uid: 180c33ea-592c-473e-b54b-692c2cc534dd + versionUid: f63c24ec-4262-4f88-ac1c-28fd47345de1 path: energy_demand/ - - uid: 84a711c4-2705-4f7a-b974-7aea05cafe0f - versionUid: 41de0075-42aa-45a1-b95a-a8d9fb81fc62 - path: lads/ - - uid: a8c81d5b-5adc-4a03-8d1b-8662829e09b1 - versionUid: 35e26802-f094-41e2-81a5-b8d8698caf81 + required: true + - name: LAD boundaries + description: > + LAD boundaries - shared definition for data exchange between models. + default: + - uid: 36adf09d-7657-4e46-a6d8-55dde43cbe51 + versionUid: 4050b797-0a12-4b86-9ca3-9edfde3aeb6d + - uid: 40ba1196-6ee1-4be3-b813-471934bf3acd + versionUid: adce60be-2019-4ec8-8f61-a297c5a26cc5 path: lads/ + required: true diff --git a/DAFNI-wrappers/et_module/model_definition.yaml b/DAFNI-wrappers/et_module/model_definition.yaml index 6f9946ee..28176135 100644 --- a/DAFNI-wrappers/et_module/model_definition.yaml +++ b/DAFNI-wrappers/et_module/model_definition.yaml @@ -65,8 +65,8 @@ spec: vehicle trip starts and consumption scenarios (test outputs from the Transport model used to test running the et_module standalone). default: - - uid: dbfe6814-3fd3-4e94-8c30-ccfec60c9989 - versionUid: df234857-a622-4b83-bec1-c72b1c7ed8b6 + - uid: ed671698-069c-40f4-8645-bf08e45634fb + versionUid: 66382de9-bef7-45ab-b729-dbfc1d01007f path: et_module/ required: true - name: LAD boundaries