From 031ad55d2b2f7a655500b72346b6221529556ff5 Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 18 Jul 2024 13:53:19 +0200 Subject: [PATCH 01/16] replace snakemake subworkflow by submodule; fix path relations --- .github/workflows/ci-linux.yaml | 2 +- .github/workflows/ci-mac.yaml | 2 +- .github/workflows/ci-windows.yaml | 2 +- .gitignore | 4 + Makefile | 19 + Snakefile | 140 +++-- config.default.yaml | 8 +- configs/powerplantmatching_config.yaml | 177 +++++++ pypsa-earth | 2 +- scripts/add_export.py | 4 +- scripts/build_base_energy_totals.py | 2 +- scripts/build_clustered_population_layouts.py | 2 +- scripts/build_heat_demand.py | 2 +- scripts/build_industrial_distribution_key.py | 2 +- scripts/build_industry_demand.py | 2 +- scripts/build_population_layouts.py | 2 +- scripts/build_ship_profile.py | 2 +- scripts/build_solar_thermal_profiles.py | 2 +- scripts/build_temperature_profiles.py | 2 +- scripts/copy_config.py | 2 +- scripts/override_respot.py | 2 +- scripts/prepare_airports.py | 2 +- scripts/prepare_energy_totals.py | 2 +- scripts/prepare_gas_network.py | 2 +- scripts/prepare_heat_data.py | 2 +- scripts/prepare_ports.py | 2 +- scripts/prepare_transport_data.py | 2 +- scripts/prepare_transport_data_input.py | 2 +- scripts/prepare_urban_percent.py | 2 +- scripts/solve_network.py | 2 +- test/config.test1.yaml | 490 +----------------- 31 files changed, 301 insertions(+), 589 deletions(-) create mode 100644 Makefile create mode 100644 configs/powerplantmatching_config.yaml diff --git a/.github/workflows/ci-linux.yaml b/.github/workflows/ci-linux.yaml index 9b6fd67e..26d59e34 100644 --- a/.github/workflows/ci-linux.yaml +++ b/.github/workflows/ci-linux.yaml @@ -68,7 +68,7 @@ jobs: - name: Test snakemake workflow run: | conda list - snakemake --cores all run_test + make test # - name: Test plotting and summaries # run: | diff --git a/.github/workflows/ci-mac.yaml b/.github/workflows/ci-mac.yaml index 6621ccd4..d4c96d26 100644 --- a/.github/workflows/ci-mac.yaml +++ b/.github/workflows/ci-mac.yaml @@ -66,7 +66,7 @@ jobs: - name: Test snakemake workflow run: | conda list - snakemake --cores all run_test + make test # - name: Test plotting and summaries # run: | diff --git a/.github/workflows/ci-windows.yaml b/.github/workflows/ci-windows.yaml index 6adb0833..15de3c36 100644 --- a/.github/workflows/ci-windows.yaml +++ b/.github/workflows/ci-windows.yaml @@ -66,7 +66,7 @@ jobs: - name: Test snakemake workflow run: | conda list - snakemake --cores all run_test + make test # - name: Test plotting and summaries # run: | diff --git a/.gitignore b/.gitignore index ff3ebbaa..753e2c9e 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,10 @@ data/costs* /data/ports.csv /data/energy_totals_DF_2030 /data/energy_totals_XX_2038 +data/ +data/osm/ +data/raw/ +data/base_network/ /data/airports.csv /data/energy_totals_base.csv diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..34510e06 --- /dev/null +++ b/Makefile @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +.PHONY: test setup clean + +test: + set -e + snakemake -call solve_all_networks --configfile pypsa-earth/config.tutorial.yaml test/config.test1.yaml + echo "All tests completed successfully." + +setup: + # Add setup commands here + echo "Setup complete." + +clean: + # Add clean-up commands here + snakemake -j1 solve_all_networks --delete-all-output --configfile test/config.test1.yaml + echo "Clean-up complete." diff --git a/Snakefile b/Snakefile index edd248c6..88dcb3cd 100644 --- a/Snakefile +++ b/Snakefile @@ -1,8 +1,11 @@ import sys -sys.path.append("pypsa-earth/scripts") + +PYPSAEARTH_FOLDER = "pypsa-earth" +sys.path.append(PYPSAEARTH_FOLDER + "/scripts") from os.path import exists +from pathlib import Path from shutil import copyfile, move from scripts.helpers import get_last_commit_message @@ -15,22 +18,23 @@ if not exists("config.yaml"): copyfile("config.default.yaml", "config.yaml") -configfile: "config.pypsa-earth.yaml" +configfile: PYPSAEARTH_FOLDER + "config.pypsa-earth.yaml" +configfile: PYPSAEARTH_FOLDER + "/configs/bundle_config.yaml" +configfile: PYPSAEARTH_FOLDER + "/configs/powerplantmatching_config.yaml" configfile: "config.yaml" -PYPSAEARTH_FOLDER = "pypsa-earth" - # convert country list according to the desired region config["countries"] = create_country_list(config["countries"]) - - -SDIR = config["summary_dir"] + config["run"] -RDIR = config["results_dir"] + config["run"] -CDIR = config["costs_dir"] - config.update({"git_commit": get_last_commit_message(".")}) config.update({"submodule_commit": get_last_commit_message(PYPSAEARTH_FOLDER)}) +config["ROOT_PATH"] = os.getcwd() + +run = config.get("run", {}) +SDIR = config["summary_dir"] + run["name"] + "/" if run.get("name") else "" +RDIR = config["results_dir"] + run["name"] + "/" if run.get("name") else "" +COSTDIR = config["costs_dir"] + CUTOUTS_PATH = ( "cutouts/cutout-2013-era5-tutorial.nc" @@ -53,24 +57,34 @@ wildcard_constraints: if not config.get("disable_subworkflow", False): - subworkflow pypsaearth: - workdir: - PYPSAEARTH_FOLDER + module pypsaearth: snakefile: PYPSAEARTH_FOLDER + "/Snakefile" - configfile: - "./config.pypsa-earth.yaml" + config: + config -if config.get("disable_subworkflow", False): + use rule * from pypsaearth - def pypsaearth(path): - return PYPSAEARTH_FOLDER + "/" + path + +data_dir = Path(PYPSAEARTH_FOLDER) / "data" +rule get_data: + output: + [ + str(Path("data") / p.relative_to(data_dir)) + for p in data_dir.rglob("*") + if p.is_file() + ], + shell: + """ + mkdir -p data + cp -nR {data_dir}/. data/ + """ if config["enable"].get("retrieve_cost_data", True): - rule retrieve_cost_data: + rule retrieve_cost_data_flexible: input: HTTP.remote( f"raw.githubusercontent.com/PyPSA/technology-data/{config['costs']['version']}/outputs/costs" @@ -78,7 +92,7 @@ if config["enable"].get("retrieve_cost_data", True): keep_local=True, ), output: - costs=CDIR + "costs_{planning_horizons}.csv", + costs=COSTDIR + "costs_{planning_horizons}.csv", resources: mem_mb=5000, run: @@ -106,7 +120,7 @@ rule override_res_all_nets: ), -rule solve_all_networks: +rule solve_sector_networks: input: expand( RDIR @@ -165,9 +179,7 @@ if not config["custom_data"]["gas_network"]: geo_crs=config["crs"]["geo_crs"], custom_gas_network=config["custom_data"]["gas_network"], input: - regions_onshore=pypsaearth( - "resources/bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson" - ), + regions_onshore="resources/bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", output: clustered_gas_network="resources/gas_networks/gas_network_elec_s{simpl}_{clusters}.csv", # TODO: Should be a own snakemake rule @@ -181,7 +193,7 @@ rule prepare_sector_network: input: network=RDIR + "/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_presec.nc", - costs=CDIR + "costs_{planning_horizons}.csv", + costs=COSTDIR + "costs_{planning_horizons}.csv", h2_cavern="data/hydrogen_salt_cavern_potentials.csv", nodal_energy_totals="resources/demand/heat/nodal_energy_heat_totals_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", transport="resources/demand/transport_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", @@ -200,9 +212,7 @@ rule prepare_sector_network: solar_thermal="resources/demand/heat/solar_thermal_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", district_heat_share="resources/demand/heat/district_heat_share_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", biomass_transport_costs="data/temp_hard_coded/biomass_transport_costs.csv", - shapes_path=pypsaearth( - "resources/bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson" - ), + shapes_path="resources/bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", pipelines="data_custom/pipelines.csv" if config["custom_data"]["gas_network"] else "resources/gas_networks/gas_network_elec_s{simpl}_{clusters}.csv", @@ -244,13 +254,11 @@ rule add_export: input: overrides="data/override_component_attrs", export_ports="data/export_ports.csv", - costs=CDIR + "costs_{planning_horizons}.csv", + costs=COSTDIR + "costs_{planning_horizons}.csv", ship_profile="resources/ship_profile_{h2export}TWh.csv", network=RDIR + "/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}.nc", - shapes_path=pypsaearth( - "resources/bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson" - ), + shapes_path="resources/bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", output: RDIR + "/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", @@ -260,7 +268,7 @@ rule add_export: rule override_respot: params: - run=config["run"], + run=run["name"], custom_data=config["custom_data"], countries=config["countries"], input: @@ -277,7 +285,7 @@ rule override_respot: for planning_horizons in config["scenario"]["planning_horizons"] }, overrides="data/override_component_attrs", - network=pypsaearth("networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"), + network="networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", energy_totals="data/energy_totals_{demand}_{planning_horizons}.csv", output: RDIR @@ -288,7 +296,7 @@ rule override_respot: rule prepare_transport_data: input: - network=pypsaearth("networks/elec_s{simpl}_{clusters}.nc"), + network="networks/elec_s{simpl}_{clusters}.nc", energy_totals_name="data/energy_totals_{demand}_{planning_horizons}.csv", traffic_data_KFZ="data/emobility/KFZ__count", traffic_data_Pkw="data/emobility/Pkw__count", @@ -332,7 +340,7 @@ rule build_cop_profiles: rule prepare_heat_data: input: - network=pypsaearth("networks/elec_s{simpl}_{clusters}.nc"), + network="networks/elec_s{simpl}_{clusters}.nc", energy_totals_name="data/energy_totals_{demand}_{planning_horizons}.csv", clustered_pop_layout="resources/population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", temp_air_total="resources/temperatures/temp_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", @@ -391,10 +399,8 @@ rule build_solar_thermal_profiles: pop_layout_total="resources/population_shares/pop_layout_total_{planning_horizons}.nc", pop_layout_urban="resources/population_shares/pop_layout_urban_{planning_horizons}.nc", pop_layout_rural="resources/population_shares/pop_layout_rural_{planning_horizons}.nc", - regions_onshore=pypsaearth( - "resources/bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson" - ), - cutout=pypsaearth(CUTOUTS_PATH), + regions_onshore="resources/bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", + cutout=CUTOUTS_PATH, output: solar_thermal_total="resources/demand/heat/solar_thermal_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", solar_thermal_urban="resources/demand/heat/solar_thermal_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", @@ -411,9 +417,9 @@ rule build_population_layouts: params: planning_horizons=config["scenario"]["planning_horizons"][0], input: - nuts3_shapes=pypsaearth("resources/shapes/gadm_shapes.geojson"), + nuts3_shapes="resources/shapes/gadm_shapes.geojson", urban_percent="data/urban_percent.csv", - cutout=pypsaearth(CUTOUTS_PATH), + cutout=CUTOUTS_PATH, output: pop_layout_total="resources/population_shares/pop_layout_total_{planning_horizons}.nc", pop_layout_urban="resources/population_shares/pop_layout_urban_{planning_horizons}.nc", @@ -443,10 +449,8 @@ rule build_clustered_population_layouts: pop_layout_urban="resources/population_shares/pop_layout_urban_{planning_horizons}.nc", pop_layout_rural="resources/population_shares/pop_layout_rural_{planning_horizons}.nc", gdp_layout="resources/gdp_shares/gdp_layout_{planning_horizons}.nc", - regions_onshore=pypsaearth( - "resources/bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson" - ), - cutout=pypsaearth(CUTOUTS_PATH), + regions_onshore="resources/bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", + cutout=CUTOUTS_PATH, output: clustered_pop_layout="resources/population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", clustered_gdp_layout="resources/gdp_shares/gdp_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", @@ -465,10 +469,8 @@ rule build_heat_demand: pop_layout_total="resources/population_shares/pop_layout_total_{planning_horizons}.nc", pop_layout_urban="resources/population_shares/pop_layout_urban_{planning_horizons}.nc", pop_layout_rural="resources/population_shares/pop_layout_rural_{planning_horizons}.nc", - regions_onshore=pypsaearth( - "resources/bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson" - ), - cutout=pypsaearth(CUTOUTS_PATH), + regions_onshore="resources/bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", + cutout=CUTOUTS_PATH, output: heat_demand_urban="resources/demand/heat/heat_demand_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", heat_demand_rural="resources/demand/heat/heat_demand_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", @@ -488,10 +490,8 @@ rule build_temperature_profiles: pop_layout_total="resources/population_shares/pop_layout_total_{planning_horizons}.nc", pop_layout_urban="resources/population_shares/pop_layout_urban_{planning_horizons}.nc", pop_layout_rural="resources/population_shares/pop_layout_rural_{planning_horizons}.nc", - regions_onshore=pypsaearth( - "resources/bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson" - ), - cutout=pypsaearth(CUTOUTS_PATH), + regions_onshore="resources/bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", + cutout=CUTOUTS_PATH, output: temp_soil_total="resources/temperatures/temp_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", temp_soil_rural="resources/temperatures/temp_soil_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", @@ -510,7 +510,7 @@ rule build_temperature_profiles: rule copy_config: params: summary_dir=config["summary_dir"], - run=config["run"], + run=run["name"], output: SDIR + "/configs/config.yaml", threads: 1 @@ -522,14 +522,14 @@ rule copy_config: "scripts/copy_config.py" -rule solve_network: +rule solve_sector_network: input: overrides="data/override_component_attrs", # network=RDIR # + "/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}.nc", network=RDIR + "/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", - costs=CDIR + "costs_{planning_horizons}.csv", + costs=COSTDIR + "costs_{planning_horizons}.csv", configs=SDIR + "/configs/config.yaml", # included to trigger copy_config rule output: RDIR @@ -555,12 +555,12 @@ rule solve_network: "scripts/solve_network.py" -rule make_summary: +rule make_sector_summary: params: planning_horizons=config["scenario"]["planning_horizons"], results_dir=config["results_dir"], summary_dir=config["summary_dir"], - run=config["run"], + run=run["name"], scenario_config=config["scenario"], costs_config=config["costs"], h2export_qty=config["export"]["h2export"], @@ -574,7 +574,7 @@ rule make_summary: **config["costs"], **config["export"] ), - costs=CDIR + "costs_{planning_horizons}.csv", + costs=COSTDIR + "costs_{planning_horizons}.csv", plots=expand( RDIR + "/maps/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}-costs-all_{planning_horizons}_{discountrate}_{demand}_{h2export}export.pdf", @@ -607,7 +607,7 @@ rule make_summary: "scripts/make_summary.py" -rule plot_network: +rule plot_sector_network: input: overrides="data/override_component_attrs", network=RDIR @@ -627,7 +627,7 @@ rule plot_network: "scripts/plot_network.py" -rule plot_summary: +rule plot_sector_summary: input: costs=SDIR + "/csvs/costs.csv", energy=SDIR + "/csvs/energy.csv", @@ -694,12 +694,6 @@ rule run_test: -rule clean: - run: - shell("rm -r " + PYPSAEARTH_FOLDER + "/resources") - shell("rm -r " + PYPSAEARTH_FOLDER + "/networks") - - rule build_industrial_distribution_key: #default data params: countries=config["countries"], @@ -707,15 +701,11 @@ rule build_industrial_distribution_key: #default data alternative_clustering=config["clustering_options"]["alternative_clustering"], industry_database=config["custom_data"]["industry_database"], input: - regions_onshore=pypsaearth( - "resources/bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson" - ), + regions_onshore="resources/bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", clustered_pop_layout="resources/population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", clustered_gdp_layout="resources/gdp_shares/gdp_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", industrial_database="data/industrial_database.csv", - shapes_path=pypsaearth( - "resources/bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson" - ), + shapes_path="resources/bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", output: industrial_distribution_key="resources/demand/industrial_distribution_key_elec_s{simpl}_{clusters}_{planning_horizons}.csv", threads: 1 @@ -760,7 +750,7 @@ rule build_industry_demand: #default data #industrial_production_per_country="data/industrial_production_per_country.csv", base_industry_totals="resources/demand/base_industry_totals_{planning_horizons}_{demand}.csv", industrial_database="data/industrial_database.csv", - costs=CDIR + "costs_{planning_horizons}.csv", + costs=COSTDIR + "costs_{planning_horizons}.csv", industry_growth_cagr="data/demand/industry_growth_cagr.csv", output: industrial_energy_demand_per_node="resources/demand/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv", diff --git a/config.default.yaml b/config.default.yaml index 08cb590e..4e54b0fe 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -5,7 +5,9 @@ results_dir: results/ summary_dir: results/ costs_dir: data/ #TODO change to the equivalent of technology data -run: test_run +run: + name: test_run + shared_cutouts: true foresight: overnight @@ -28,6 +30,10 @@ scenario: demand: - "AB" +electricity: + extendable_carriers: + Link: [] + policy_config: hydrogen: temporal_matching: "no_res_matching" #either "h2_yearly_matching", "h2_monthly_matching", "no_res_matching" diff --git a/configs/powerplantmatching_config.yaml b/configs/powerplantmatching_config.yaml new file mode 100644 index 00000000..86ac5948 --- /dev/null +++ b/configs/powerplantmatching_config.yaml @@ -0,0 +1,177 @@ +# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +# ---------------------------------------------------------------------------- # +# IO Config # +# ---------------------------------------------------------------------------- # +entsoe_token: +google_api_key: + +# ---------------------------------------------------------------------------- # +# Matching Config # +# ---------------------------------------------------------------------------- # + +#matching config +matching_sources: + # - CARMA # deprecated as no more public +- GEO +- GPD +- GBPT +- GGPT +- GCPT +- GGTPT +- GNPT +- GSPT +- GWPT +- GHPT + +# - GEM_GGPT # to be enabled once GEM_GGPT is working +# - EXTERNAL_DATABASE + +fully_included_sources: + # - CARMA # deprecated as no more public +- GEO +- GPD +- GBPT +- GGPT +- GCPT +- GGTPT +- GNPT +- GSPT +- GWPT +- GHPT +# - GEM_GGPT # to be enabled once GEM_GGPT is working +# - EXTERNAL_DATABASE + +EXTERNAL_DATABASE: + reliability_score: 5 + store_locally: false + fn: /data/davidef/git/pypsa-earth/resources/powerplants_osm2pm.csv # MODIFIED_BY_WORKFLOW + +parallel_duke_processes: false +process_limit: 2 +remove_missing_coords: true + +matched_data_url: + +display_net_caps: true + +target_columns: +- Name +- Fueltype +- Technology +- Set +- Country +- Capacity +- Efficiency +- Duration +- Volume_Mm3 +- DamHeight_m +- StorageCapacity_MWh +- DateIn +- DateRetrofit +- DateOut +- lat +- lon +- EIC +- projectID + +target_fueltypes: + # Provide a mapping of the keys to a list or a regex expression which are used for parsing. + # A list will be converted to a regex expression matching all words (case-insensitive) + # given by the list. An empty string results in a regex expression containing only the key. + # Parsed of representatives at the top may be overwritten by representatives further below. + Other: ".*" + Bioenergy: [biological, biogas, bioenergy, agricultural, wood, biomass] + Nuclear: [nuclear] + Natural Gas: [ccgt, gas, natural gas, ocgt, lng, combined cycle, fossil gas, mixed fossil fuels] + Hydro: [run-off, run off, run of river, run-of-river, ror, hydro, hydroelectric, wasserkraft] + Hard Coal: [coal, coke] + Lignite: [brown coal, lignite, peat] + Oil: [oil, diesel] + Geothermal: "" + Solar: "" + Waste: "" + Wind: "" + Battery: [Electro-chemical, battery] +target_sets: + # Provide a mapping of the keys to a list or a regex expression which are used for parsing. + # A list will be converted to a regex expression matching all words (case-insensitive) + # given by the list. An empty string results in a regex expression containing only the key. + # Parsed of representatives at the top may be overwritten by representatives further below. + PP: ".*" + CHP: [heizkraftwerk, hkw, kwk, fhkw, gud, hp, bhkw, cogeneration, power and heat, heat and power, chp] + Store: [battery, storage, store] +target_technologies: + # Provide a mapping of the keys to a list or a regex expression which are used for parsing. + # A list will be converted to a regex expression matching all words (case-insensitive) + # given by the list. An empty string results in a regex expression containing only the key. + # Parsed of representatives at the top may be overwritten by representatives further below. + CCGT: [ccgt, gas, natural gas] + OCGT: [ocgt] + Steam Turbine: [steam, turbine] + Combustion Engine: [combustion engine] + Run-Of-River: [run-off, run off, run of river, run-of-river, ror] + Pumped Storage: [pumped hydro, pumped] + Reservoir: "" + Marine: "" + Onshore: "" + Offshore: "" + PV: [pv, photo-voltaic, photo voltaic] + CSP: "" +clean_name: + remove_common_words: false # remove words which appear more that 20 times in all entries + remove_duplicated_words: true + replace: + " ": "[^a-zA-Z]" # non-alphabetical symbols + "": [I, II, III, IV, V, VI, VII, VIII, IX, X, XI, parque, grupo, station, power, plant, unit, kraftwerk, kw, hkw, nuclear, thermal, heizkraftwerk, eolico, project, hydroelectric, pumped, storage, france, austria, sweden, serbia, ukraine, switzerland, slovakia, croatia, poland, slovenia, portugal, bosnia, and, herzegovina, \w] + "ss": "ß" + +# ---------------------------------------------------------------------------- # +# Heuristic Config # +# ---------------------------------------------------------------------------- # +fuel_to_lifetime: + Bioenergy: 20 + Geothermal: 15 + Hard Coal: 45 + Hydro: 100 + Lignite: 45 + Natural Gas: 40 + Nuclear: 50 + Oil: 40 + Other: 5 + Solar: 25 + Waste: 25 + Wind: 25 + +# ---------------------------------------------------------------------------- # +# Plotting Config # +# ---------------------------------------------------------------------------- # +fuel_to_color: + OCGT: darkorange + Hydro: seagreen + Pumped hydro: limegreen + Run-of-river: navy + Ror: navy + Lignite: saddlebrown + Nuclear: yellow + Solar: gold + Windoff: cornflowerblue + Windon: darkblue + Offshore: cornflowerblue + Onshore: darkblue + Wind: steelblue + Bioenergy: forestgreen + Natural Gas: red + CCGT: red + Coal: black + Hard Coal: dimgray + Oil: black + Other: silver + Waste: grey + Geothermal: orange + Battery: purple + Hydrogen Storage: teal + Electro-mechanical: teal + Total: gold diff --git a/pypsa-earth b/pypsa-earth index de4da143..865cadfc 160000 --- a/pypsa-earth +++ b/pypsa-earth @@ -1 +1 @@ -Subproject commit de4da143e92d4d7aefcaac588175db82d93f0670 +Subproject commit 865cadfc5ce01065c9daac8f01f2d4dfa50e2391 diff --git a/scripts/add_export.py b/scripts/add_export.py index f73b48ca..a97d6c07 100644 --- a/scripts/add_export.py +++ b/scripts/add_export.py @@ -177,7 +177,7 @@ def create_export_profile(): if __name__ == "__main__": if "snakemake" not in globals(): os.chdir(os.path.dirname(os.path.abspath(__file__))) - from helpers import mock_snakemake, sets_path_to_root + from helpers import mock_snakemake snakemake = mock_snakemake( "add_export", @@ -191,7 +191,7 @@ def create_export_profile(): demand="AB", h2export="120", ) - sets_path_to_root("pypsa-earth-sec") + os.chdir(snakemake.config["ROOT_PATH"]) overrides = override_component_attrs(snakemake.input.overrides) n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) diff --git a/scripts/build_base_energy_totals.py b/scripts/build_base_energy_totals.py index 951fbea4..b3d60eb6 100644 --- a/scripts/build_base_energy_totals.py +++ b/scripts/build_base_energy_totals.py @@ -261,7 +261,7 @@ def calc_sector(sector): demand="DF", planning_horizons=2030, ) - sets_path_to_root("pypsa-earth-sec") + os.chdir(snakemake.config["ROOT_PATH"]) energy_stat_database = pd.read_excel( snakemake.input.unsd_paths, index_col=0, header=0 diff --git a/scripts/build_clustered_population_layouts.py b/scripts/build_clustered_population_layouts.py index b08facca..a0531e01 100644 --- a/scripts/build_clustered_population_layouts.py +++ b/scripts/build_clustered_population_layouts.py @@ -18,7 +18,7 @@ simpl="", clusters=38, ) - sets_path_to_root("pypsa-earth-sec") + os.chdir(snakemake.config["ROOT_PATH"]) cutout_path = ( snakemake.input.cutout diff --git a/scripts/build_heat_demand.py b/scripts/build_heat_demand.py index babc3364..f75c3795 100644 --- a/scripts/build_heat_demand.py +++ b/scripts/build_heat_demand.py @@ -15,7 +15,7 @@ os.chdir(os.path.dirname(os.path.abspath(__file__))) snakemake = mock_snakemake("build_heat_demand", simpl="", clusters="10") - sets_path_to_root("pypsa-earth-sec") + os.chdir(snakemake.config["ROOT_PATH"]) time = pd.date_range(freq="h", **snakemake.params.snapshots) cutout_config = snakemake.input.cutout diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py index e8e6b258..cb5d621e 100644 --- a/scripts/build_industrial_distribution_key.py +++ b/scripts/build_industrial_distribution_key.py @@ -121,7 +121,7 @@ def match_technology(df): demand="AB", planning_horizons=2050, ) - sets_path_to_root("pypsa-earth-sec") + os.chdir(snakemake.config["ROOT_PATH"]) regions = gpd.read_file(snakemake.input.regions_onshore) shapes_path = snakemake.input.shapes_path diff --git a/scripts/build_industry_demand.py b/scripts/build_industry_demand.py index 46148c57..5b8cfa8b 100644 --- a/scripts/build_industry_demand.py +++ b/scripts/build_industry_demand.py @@ -62,7 +62,7 @@ def country_to_nodal(industrial_production, keys): demand="AB", ) - sets_path_to_root("pypsa-earth-sec") + os.chdir(snakemake.config["ROOT_PATH"]) countries = snakemake.params.countries diff --git a/scripts/build_population_layouts.py b/scripts/build_population_layouts.py index 00f76fff..525c76cd 100644 --- a/scripts/build_population_layouts.py +++ b/scripts/build_population_layouts.py @@ -20,7 +20,7 @@ "build_population_layouts", planning_horizons=2030, ) - sets_path_to_root("pypsa-earth-sec") + os.chdir(snakemake.config["ROOT_PATH"]) cutout_path = ( snakemake.input.cutout diff --git a/scripts/build_ship_profile.py b/scripts/build_ship_profile.py index 0d1c9c92..0c5f78ae 100644 --- a/scripts/build_ship_profile.py +++ b/scripts/build_ship_profile.py @@ -69,7 +69,7 @@ def build_ship_profile(export_volume, ship_opts): "build_ship_profile", h2export="120", ) - sets_path_to_root("pypsa-earth-sec") + os.chdir(snakemake.config["ROOT_PATH"]) # Get parameters from config and wildcard ship_opts = snakemake.params.ship_opts diff --git a/scripts/build_solar_thermal_profiles.py b/scripts/build_solar_thermal_profiles.py index 9ddba06e..71ad2b67 100644 --- a/scripts/build_solar_thermal_profiles.py +++ b/scripts/build_solar_thermal_profiles.py @@ -19,7 +19,7 @@ simpl="", clusters=15, ) - sets_path_to_root("pypsa-earth-sec") + os.chdir(snakemake.config["ROOT_PATH"]) config = snakemake.params.solar_thermal_config diff --git a/scripts/build_temperature_profiles.py b/scripts/build_temperature_profiles.py index c32c69f7..2b6679d0 100644 --- a/scripts/build_temperature_profiles.py +++ b/scripts/build_temperature_profiles.py @@ -18,7 +18,7 @@ simpl="", clusters=900, ) - sets_path_to_root("pypsa-earth-sec") + os.chdir(snakemake.config["ROOT_PATH"]) time = pd.date_range(freq="h", **snakemake.params.snapshots) cutout_path = ( diff --git a/scripts/copy_config.py b/scripts/copy_config.py index b5c44b0a..fd5a929b 100644 --- a/scripts/copy_config.py +++ b/scripts/copy_config.py @@ -17,7 +17,7 @@ os.chdir(os.path.dirname(os.path.abspath(__file__))) snakemake = mock_snakemake("copy_config") - sets_path_to_root("pypsa-earth-sec") + os.chdir(snakemake.config["ROOT_PATH"]) for f, name in files_to_copy.items(): copy( diff --git a/scripts/override_respot.py b/scripts/override_respot.py index 3b65f7bb..e7d5943e 100644 --- a/scripts/override_respot.py +++ b/scripts/override_respot.py @@ -80,7 +80,7 @@ def override_values(tech, year, dr): demand="AP", discountrate=0.071, ) - sets_path_to_root("pypsa-earth-sec") + os.chdir(snakemake.config["ROOT_PATH"]) overrides = override_component_attrs(snakemake.input.overrides) n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) diff --git a/scripts/prepare_airports.py b/scripts/prepare_airports.py index 7bfd24b5..0d40b76a 100644 --- a/scripts/prepare_airports.py +++ b/scripts/prepare_airports.py @@ -40,7 +40,7 @@ def download_airports(): os.chdir(os.path.dirname(os.path.abspath(__file__))) snakemake = mock_snakemake("prepare_airports") - sets_path_to_root("pypsa-earth-sec") + os.chdir(snakemake.config["ROOT_PATH"]) # configure_logging(snakemake) # run = snakemake.config.get("run", {}) diff --git a/scripts/prepare_energy_totals.py b/scripts/prepare_energy_totals.py index 571ce626..94b6fd0f 100644 --- a/scripts/prepare_energy_totals.py +++ b/scripts/prepare_energy_totals.py @@ -44,7 +44,7 @@ def calculate_end_values(df): demand="EG", planning_horizons=2030, ) - sets_path_to_root("pypsa-earth-sec") + os.chdir(snakemake.config["ROOT_PATH"]) countries = snakemake.params.countries # countries = ["NG", "BJ"] diff --git a/scripts/prepare_gas_network.py b/scripts/prepare_gas_network.py index f48ca684..f8833b3a 100644 --- a/scripts/prepare_gas_network.py +++ b/scripts/prepare_gas_network.py @@ -38,7 +38,7 @@ simpl="", clusters="10", ) - sets_path_to_root("pypsa-earth-sec") + os.chdir(snakemake.config["ROOT_PATH"]) rootpath = ".." else: rootpath = "." diff --git a/scripts/prepare_heat_data.py b/scripts/prepare_heat_data.py index 99d3397a..06e2801b 100644 --- a/scripts/prepare_heat_data.py +++ b/scripts/prepare_heat_data.py @@ -139,7 +139,7 @@ def prepare_heat_data(n): planning_horizons=2030, demand="DF", ) - sets_path_to_root("pypsa-earth-sec") + os.chdir(snakemake.config["ROOT_PATH"]) n = pypsa.Network(snakemake.input.network) diff --git a/scripts/prepare_ports.py b/scripts/prepare_ports.py index 5717a08f..db9e1804 100644 --- a/scripts/prepare_ports.py +++ b/scripts/prepare_ports.py @@ -31,7 +31,7 @@ def download_ports(): os.chdir(os.path.dirname(os.path.abspath(__file__))) snakemake = mock_snakemake("prepare_ports") - sets_path_to_root("pypsa-earth-sec") + os.chdir(snakemake.config["ROOT_PATH"]) # configure_logging(snakemake) # run = snakemake.config.get("run", {}) diff --git a/scripts/prepare_transport_data.py b/scripts/prepare_transport_data.py index 9eaa86ed..ce91dedd 100644 --- a/scripts/prepare_transport_data.py +++ b/scripts/prepare_transport_data.py @@ -210,7 +210,7 @@ def prepare_transport_data(n): planning_horizons=2030, ) - sets_path_to_root("pypsa-earth-sec") + os.chdir(snakemake.config["ROOT_PATH"]) n = pypsa.Network(snakemake.input.network) diff --git a/scripts/prepare_transport_data_input.py b/scripts/prepare_transport_data_input.py index e553b48a..0202a672 100644 --- a/scripts/prepare_transport_data_input.py +++ b/scripts/prepare_transport_data_input.py @@ -95,7 +95,7 @@ def download_CO2_emissions(): os.chdir(os.path.dirname(os.path.abspath(__file__))) snakemake = mock_snakemake("prepare_transport_data_input") - sets_path_to_root("pypsa-earth-sec") + os.chdir(snakemake.config["ROOT_PATH"]) # configure_logging(snakemake) # run = snakemake.config.get("run", {}) diff --git a/scripts/prepare_urban_percent.py b/scripts/prepare_urban_percent.py index dc0cd1c4..d174f003 100644 --- a/scripts/prepare_urban_percent.py +++ b/scripts/prepare_urban_percent.py @@ -71,7 +71,7 @@ def download_urban_percent(): os.chdir(os.path.dirname(os.path.abspath(__file__))) snakemake = mock_snakemake("prepare_urban_percent") - sets_path_to_root("pypsa-earth-sec") + os.chdir(snakemake.config["ROOT_PATH"]) # configure_logging(snakemake) # run = snakemake.config.get("run", {}) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 20138240..d383836a 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -561,7 +561,7 @@ def add_existing(n): h2export="0", ) - sets_path_to_root("pypsa-earth-sec") + os.chdir(snakemake.config["ROOT_PATH"]) logging.basicConfig( filename=snakemake.log.python, level=snakemake.config["logging_level"] diff --git a/test/config.test1.yaml b/test/config.test1.yaml index ba2ce555..e55fd826 100644 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -1,514 +1,30 @@ -logging_level: INFO tutorial: true -results_dir: results/ -summary_dir: results/ -costs_dir: data/ #TODO change to the equivalent of technology data - -run: test3 - -foresight: overnight - -# option to disable the subworkflow to ease the analyses -disable_subworkflow: false +run: + name: test1 + shared_cutouts: true scenario: - simpl: # only relevant for PyPSA-Eur - - "" clusters: # number of nodes in Europe, any integer between 37 (1 node per country-zone) and several hundred - 4 - planning_horizons: # investment years for myopic and perfect; or costs year for overnight - - 2030 ll: - "c1" - opts: - - "Co2L" - sopts: - - "144H" - demand: - - "AB" - -policy_config: - hydrogen: - temporal_matching: "no_res_matching" #either "h2_yearly_matching", "h2_monthly_matching", "no_res_matching" - spatial_matching: false - additionality: false # RE electricity is equal to the amount required for additional hydrogen export compared to the 0 export case ("reference_case") - allowed_excess: 1.0 - is_reference: false # Whether or not this network is a reference case network, relevant only if additionality is _true_ - remove_h2_load: false #Whether or not to remove the h2 load from the network, relevant only if is_reference is _true_ - path_to_ref: "" # Path to the reference case network for additionality calculation, relevant only if additionality is _true_ and is_reference is _false_ - re_country_load: false # Set to "True" to force the RE electricity to be equal to the electricity required for hydrogen export and the country electricity load. "False" excludes the country electricity load from the constraint. - -clustering_options: - alternative_clustering: true countries: ['NG', 'BJ'] -demand_data: - update_data: true # if true, the workflow downloads the energy balances data saved in data/demand/unsd/data again. Turn on for the first run. - base_year: 2019 - - other_industries: false # Whether or not to include industries that are not specified. some countries have has exageratted numbers, check carefully. - aluminium_year: 2019 # Year of the aluminium demand data specified in `data/AL_production.csv` - - -enable: - retrieve_cost_data: true # if true, the workflow overwrites the cost data saved in data/costs again - -fossil_reserves: - oil: 100 #TWh Maybe reduntant - - export: h2export: [120] # Yearly export demand in TWh store: true # [True, False] # specifies wether an export store to balance demand is implemented store_capital_costs: "no_costs" # ["standard_costs", "no_costs"] # specifies the costs of the export store "standard_costs" takes CAPEX of "hydrogen storage tank type 1 including compressor" - export_profile: "ship" # use "ship" or "constant" - ship: - ship_capacity: 0.4 # TWh # 0.05 TWh for new ones, 0.003 TWh for Susio Frontier, 0.4 TWh according to Hampp2021: "Corresponds to 11360 t H2 (l) with LHV of 33.3333 Mwh/t_H2. Cihlar et al 2020 based on IEA 2019, Table 3-B" - travel_time: 288 # hours # From Agadir to Rotterdam and back (12*24) - fill_time: 24 # hours, for 48h see Hampp2021 - unload_time: 24 # hours for 48h see Hampp2021 - - -custom_data: - renewables: [] # ['csp', 'rooftop-solar', 'solar'] - elec_demand: false - heat_demand: false - industry_demand: false - industry_database: false - transport_demand: false - water_costs: false - h2_underground: false - add_existing: false - custom_sectors: false - gas_network: false # If "True" then a custom .csv file must be placed in "resources/custom_data/pipelines.csv" , If "False" the user can choose btw "greenfield" or Model built-in datasets. Please refer to ["sector"] below. - -costs: # Costs used in PyPSA-Earth-Sec. Year depends on the wildcard planning_horizon in the scenario section - version: v0.6.2 - lifetime: 25 #default lifetime - # From a Lion Hirth paper, also reflects average of Noothout et al 2016 - discountrate: [0.071] #, 0.086, 0.111] - # [EUR/USD] ECB: https://www.ecb.europa.eu/stats/exchange/eurofxref/html/eurofxref-graph-usd.en.html # noqa: E501 - USD2013_to_EUR2013: 0.7532 - # Marginal and capital costs can be overwritten - # capital_cost: - # onwind: 500 - marginal_cost: - solar: 0.01 - onwind: 0.015 - offwind: 0.015 - hydro: 0. - H2: 0. - battery: 0. - - - emission_prices: # only used with the option Ep (emission prices) - co2: 0. - - lines: - length_factor: 1.25 #to estimate offwind connection costs - - -industry: - reference_year: 2015 - -solar_thermal: - clearsky_model: simple - orientation: - slope: 45. - azimuth: 180. sector: - gas: - spatial_gas: true # ALWAYS TRUE - network: false # ALWAYS FALSE for now (NOT USED) - network_data: GGIT # Global dataset -> 'GGIT' , European dataset -> 'IGGIELGN' - network_data_GGIT_status: ['Construction', 'Operating', 'Idle', 'Shelved', 'Mothballed', 'Proposed'] - hydrogen: - network: true - network_limit: 2000 #GWkm - network_routes: gas # "gas or "greenfield". If "gas" -> the network data are fetched from ["sector"]["gas"]["network_data"]. If "greenfield" -> the network follows the topology of electrical transmission lines - gas_network_repurposing: true # If true -> ["sector"]["gas"]["network"] is automatically false - underground_storage: false - hydrogen_colors: false - set_color_shares: false - blue_share: 0.40 - pink_share: 0.05 - - international_bunkers: false #Whether or not to count the emissions of international aviation and navigation - - oil: - spatial_oil: true - - district_heating: - potential: 0.3 #maximum fraction of urban demand which can be supplied by district heating - #increase of today's district heating demand to potential maximum district heating share - #progress = 0 means today's district heating share, progress=-1 means maxumzm fraction of urban demand is supplied by district heating - progress: 1 - #2020: 0.0 - #2030: 0.3 - #2040: 0.6 - #2050: 1.0 - district_heating_loss: 0.15 - reduce_space_heat_exogenously: true # reduces space heat demand by a given factor (applied before losses in DH) - # this can represent e.g. building renovation, building demolition, or if - # the factor is negative: increasing floor area, increased thermal comfort, population growth - reduce_space_heat_exogenously_factor: 0.29 # per unit reduction in space heat demand - # the default factors are determined by the LTS scenario from http://tool.european-calculator.eu/app/buildings/building-types-area/?levers=1ddd4444421213bdbbbddd44444ffffff11f411111221111211l212221 - # 2020: 0.10 # this results in a space heat demand reduction of 10% - # 2025: 0.09 # first heat demand increases compared to 2020 because of larger floor area per capita - # 2030: 0.09 - # 2035: 0.11 - # 2040: 0.16 - # 2045: 0.21 - # 2050: 0.29 - - tes: true - tes_tau: # 180 day time constant for centralised, 3 day for decentralised - decentral: 3 - central: 180 - boilers: true - oil_boilers: false - chp: true - micro_chp: false - solar_thermal: true - heat_pump_sink_T: 55 #Celsius, based on DTU / large area radiators; used un build_cop_profiles.py - time_dep_hp_cop: true #time dependent heat pump coefficient of performance - solar_cf_correction: 0.788457 # = >>>1/1.2683 - bev_plug_to_wheel_efficiency: 0.2 #kWh/km from EPA https://www.fueleconomy.gov/feg/ for Tesla Model S - bev_charge_efficiency: 0.9 #BEV (dis-)charging efficiency - transport_heating_deadband_upper: 20. - transport_heating_deadband_lower: 15. - ICE_lower_degree_factor: 0.375 #in per cent increase in fuel consumption per degree above deadband - ICE_upper_degree_factor: 1.6 - EV_lower_degree_factor: 0.98 - EV_upper_degree_factor: 0.63 - bev_avail_max: 0.95 - bev_avail_mean: 0.8 - bev_dsm_restriction_value: 0.75 #Set to 0 for no restriction on BEV DSM - bev_dsm_restriction_time: 7 #Time at which SOC of BEV has to be dsm_restriction_value - v2g: true #allows feed-in to grid from EV battery - bev_dsm: true #turns on EV battery - bev_energy: 0.05 #average battery size in MWh - bev_availability: 0.5 #How many cars do smart charging - transport_fuel_cell_efficiency: 0.5 - transport_internal_combustion_efficiency: 0.3 - industry_util_factor: 0.7 - - biomass_transport: true # biomass transport between nodes - biomass_transport_default_cost: 0.1 #EUR/km/MWh solid_biomass_potential: 10 # TWh/a, Potential of whole modelled area - biogas_potential: 0.5 # TWh/a, Potential of whole modelled area - efficiency_heat_oil_to_elec: 0.9 - efficiency_heat_biomass_to_elec: 0.9 - efficiency_heat_gas_to_elec: 0.9 - - dynamic_transport: - enable: false # If "True", then the BEV and FCEV shares are obtained depening on the "Co2L"-wildcard (e.g. "Co2L0.70: 0.10"). If "False", then the shares are obtained depending on the "demand" wildcard and "planning_horizons" wildcard as listed below (e.g. "DF_2050: 0.08") - land_transport_electric_share: - Co2L2.0: 0.00 - Co2L1.0: 0.01 - Co2L0.90: 0.03 - Co2L0.80: 0.06 - Co2L0.70: 0.10 - Co2L0.60: 0.17 - Co2L0.50: 0.27 - Co2L0.40: 0.40 - Co2L0.30: 0.55 - Co2L0.20: 0.69 - Co2L0.10: 0.80 - Co2L0.00: 0.88 - land_transport_fuel_cell_share: - Co2L2.0: 0.01 - Co2L1.0: 0.01 - Co2L0.90: 0.01 - Co2L0.80: 0.01 - Co2L0.70: 0.01 - Co2L0.60: 0.01 - Co2L0.50: 0.01 - Co2L0.40: 0.01 - Co2L0.30: 0.01 - Co2L0.20: 0.01 - Co2L0.10: 0.01 - Co2L0.00: 0.01 - - land_transport_fuel_cell_share: # 1 means all FCEVs HERE - BU_2030: 0.00 - AP_2030: 0.004 - NZ_2030: 0.02 - DF_2030: 0.01 - AB_2030: 0.01 - BU_2050: 0.00 - AP_2050: 0.06 - NZ_2050: 0.28 - DF_2050: 0.08 - - land_transport_electric_share: # 1 means all EVs # This leads to problems when non-zero HERE - BU_2030: 0.00 - AP_2030: 0.075 - NZ_2030: 0.13 - DF_2030: 0.01 - AB_2030: 0.01 - BU_2050: 0.00 - AP_2050: 0.42 - NZ_2050: 0.68 - DF_2050: 0.011 - - co2_network: true - co2_sequestration_potential: 200 #MtCO2/a sequestration potential for Europe - co2_sequestration_cost: 10 #EUR/tCO2 for sequestration of CO2 - hydrogen_underground_storage: true - shipping_hydrogen_liquefaction: false - shipping_average_efficiency: 0.4 #For conversion of fuel oil to propulsion in 2011 - - shipping_hydrogen_share: #1.0 - BU_2030: 0.00 - AP_2030: 0.00 - NZ_2030: 0.10 - DF_2030: 0.05 - AB_2030: 0.05 - BU_2050: 0.00 - AP_2050: 0.25 - NZ_2050: 0.36 - DF_2050: 0.12 - gadm_level: 2 - h2_cavern: true - marginal_cost_storage: 0 - methanation: true - helmeth: true - dac: true - SMR: true - SMR CC: true - cc_fraction: 0.9 - cc: true - space_heat_share: 0.6 # the share of space heating from all heating. Remainder goes to water heating. - airport_sizing_factor: 3 - - min_part_load_fischer_tropsch: 0.9 - - conventional_generation: # generator : carrier - OCGT: gas - #Gen_Test: oil # Just for testing purposes - -# snapshots are originally set in PyPSA-Earth/config.yaml but used again by PyPSA-Earth-Sec snapshots: # arguments to pd.date_range start: "2013-03-1" end: "2013-03-7" - inclusive: "left" # end is not inclusive - -# atlite: -# cutout: ./cutouts/africa-2013-era5.nc - -build_osm_network: # TODO: To Remove this once we merge pypsa-earth and pypsa-earth-sec - force_ac: false # When true, it forces all components (lines and substation) to be AC-only. To be used if DC assets create problem. solving: - #tmpdir: "path/to/tmp" - options: - formulation: kirchhoff - clip_p_max_pu: 1.e-2 - load_shedding: false - noisy_costs: true - skip_iterations: true - track_iterations: false - min_iterations: 4 - max_iterations: 6 - solver: name: cbc - - mem: 30000 #memory in MB; 20 GB enough for 50+B+I+H2; 100 GB for 181+B+I+H2 - -plotting: - map: - boundaries: [-11, 30, 34, 71] - color_geomap: - ocean: white - land: whitesmoke - costs_max: 10 - costs_threshold: 0.2 - energy_max: 20000 - energy_min: -20000 - energy_threshold: 15 - vre_techs: - - onwind - - offwind-ac - - offwind-dc - - solar - - ror - renewable_storage_techs: - - PHS - - hydro - conv_techs: - - OCGT - - CCGT - - Nuclear - - Coal - storage_techs: - - hydro+PHS - - battery - - H2 - load_carriers: - - AC load - AC_carriers: - - AC line - - AC transformer - link_carriers: - - DC line - - Converter AC-DC - heat_links: - - heat pump - - resistive heater - - CHP heat - - CHP electric - - gas boiler - - central heat pump - - central resistive heater - - central CHP heat - - central CHP electric - - central gas boiler - heat_generators: - - gas boiler - - central gas boiler - - solar thermal collector - - central solar thermal collector - tech_colors: - SMR CC: "darkblue" - gas for industry CC: "brown" - process emissions CC: "gray" - CO2 pipeline: "gray" - onwind: "dodgerblue" - onshore wind: "#235ebc" - offwind: "#6895dd" - offshore wind: "#6895dd" - offwind-ac: "c" - offshore wind (AC): "#6895dd" - offwind-dc: "#74c6f2" - offshore wind (DC): "#74c6f2" - wave: '#004444' - hydro: '#3B5323' - hydro reservoir: '#3B5323' - ror: '#78AB46' - run of river: '#78AB46' - hydroelectricity: 'blue' - solar: "orange" - solar PV: "#f9d002" - solar thermal: coral - solar rooftop: '#ffef60' - OCGT: wheat - OCGT marginal: sandybrown - OCGT-heat: '#ee8340' - gas boiler: '#ee8340' - gas boilers: '#ee8340' - gas boiler marginal: '#ee8340' - gas-to-power/heat: 'brown' - gas: brown - natural gas: brown - SMR: '#4F4F2F' - oil: '#B5A642' - oil boiler: '#B5A677' - lines: k - transmission lines: k - H2: m - H2 liquefaction: m - hydrogen storage: m - battery: slategray - battery storage: slategray - home battery: '#614700' - home battery storage: '#614700' - Nuclear: r - Nuclear marginal: r - nuclear: r - uranium: r - Coal: k - coal: k - Coal marginal: k - Lignite: grey - lignite: grey - Lignite marginal: grey - CCGT: '#ee8340' - CCGT marginal: '#ee8340' - heat pumps: '#76EE00' - heat pump: '#76EE00' - air heat pump: '#76EE00' - ground heat pump: '#40AA00' - power-to-heat: 'red' - resistive heater: pink - Sabatier: '#FF1493' - methanation: '#FF1493' - power-to-gas: 'purple' - power-to-liquid: 'darkgreen' - helmeth: '#7D0552' - DAC: 'deeppink' - co2 stored: '#123456' - CO2 sequestration: '#123456' - CC: k - co2: '#123456' - co2 vent: '#654321' - agriculture heat: '#D07A7A' - agriculture machinery oil: '#1e1e1e' - agriculture machinery oil emissions: '#111111' - agriculture electricity: '#222222' - solid biomass for industry co2 from atmosphere: '#654321' - solid biomass for industry co2 to stored: '#654321' - solid biomass for industry CC: '#654321' - gas for industry co2 to atmosphere: '#654321' - gas for industry co2 to stored: '#654321' - Fischer-Tropsch: '#44DD33' - kerosene for aviation: '#44BB11' - naphtha for industry: '#44FF55' - land transport oil: '#44DD33' - water tanks: '#BBBBBB' - hot water storage: '#BBBBBB' - hot water charging: '#BBBBBB' - hot water discharging: '#999999' - # CO2 pipeline: '#999999' - CHP: r - CHP heat: r - CHP electric: r - PHS: g - Ambient: k - Electric load: b - Heat load: r - heat: darkred - rural heat: '#880000' - central heat: '#b22222' - decentral heat: '#800000' - low-temperature heat for industry: '#991111' - process heat: '#FF3333' - heat demand: darkred - electric demand: k - Li ion: grey - district heating: '#CC4E5C' - retrofitting: purple - building retrofitting: purple - BEV charger: grey - V2G: grey - land transport EV: grey - electricity: k - gas for industry: '#333333' - solid biomass for industry: '#555555' - industry electricity: '#222222' - industry new electricity: '#222222' - process emissions to stored: '#444444' - process emissions to atmosphere: '#888888' - process emissions: '#222222' - oil emissions: '#666666' - industry oil emissions: '#666666' - land transport oil emissions: '#666666' - land transport fuel cell: '#AAAAAA' - biogas: '#800000' - solid biomass: '#DAA520' - today: '#D2691E' - shipping: '#6495ED' - shipping oil: "#6495ED" - shipping oil emissions: "#6495ED" - electricity distribution grid: 'y' - solid biomass transport: green - H2 for industry: "#222222" - H2 for shipping: "#6495ED" - biomass EOP: "green" - high-temp electrolysis: "magenta" From 443cd35c01f67e9a17df20a0e5d7397157d531bb Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 19 Jul 2024 11:55:27 +0200 Subject: [PATCH 02/16] snakefile: replace rdir by resdir for compatibility with pypsa-earth --- Makefile | 4 +-- Snakefile | 68 ++++++++++++++++++++++----------------------- config.default.yaml | 2 +- 3 files changed, 37 insertions(+), 37 deletions(-) diff --git a/Makefile b/Makefile index 34510e06..3729944f 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ test: set -e - snakemake -call solve_all_networks --configfile pypsa-earth/config.tutorial.yaml test/config.test1.yaml + snakemake -call solve_sector_networks --configfile pypsa-earth/config.tutorial.yaml test/config.test1.yaml echo "All tests completed successfully." setup: @@ -15,5 +15,5 @@ setup: clean: # Add clean-up commands here - snakemake -j1 solve_all_networks --delete-all-output --configfile test/config.test1.yaml + snakemake -j1 solve_sector_networks --delete-all-output --configfile test/config.test1.yaml echo "Clean-up complete." diff --git a/Snakefile b/Snakefile index 88dcb3cd..9a3f1008 100644 --- a/Snakefile +++ b/Snakefile @@ -32,7 +32,7 @@ config["ROOT_PATH"] = os.getcwd() run = config.get("run", {}) SDIR = config["summary_dir"] + run["name"] + "/" if run.get("name") else "" -RDIR = config["results_dir"] + run["name"] + "/" if run.get("name") else "" +RESDIR = config["results_dir"] + run["name"] + "/" if run.get("name") else "" COSTDIR = config["costs_dir"] @@ -102,8 +102,8 @@ if config["enable"].get("retrieve_cost_data", True): rule prepare_sector_networks: input: expand( - RDIR - + "/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}.nc", + RESDIR + + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}.nc", **config["scenario"], **config["costs"] ), @@ -112,8 +112,8 @@ rule prepare_sector_networks: rule override_res_all_nets: input: expand( - RDIR - + "/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_presec.nc", + RESDIR + + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_presec.nc", **config["scenario"], **config["costs"], **config["export"] @@ -123,7 +123,7 @@ rule override_res_all_nets: rule solve_sector_networks: input: expand( - RDIR + RESDIR + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", **config["scenario"], **config["costs"], @@ -191,8 +191,8 @@ if not config["custom_data"]["gas_network"]: rule prepare_sector_network: input: - network=RDIR - + "/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_presec.nc", + network=RESDIR + + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_presec.nc", costs=COSTDIR + "costs_{planning_horizons}.csv", h2_cavern="data/hydrogen_salt_cavern_potentials.csv", nodal_energy_totals="resources/demand/heat/nodal_energy_heat_totals_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", @@ -217,14 +217,14 @@ rule prepare_sector_network: if config["custom_data"]["gas_network"] else "resources/gas_networks/gas_network_elec_s{simpl}_{clusters}.csv", output: - RDIR - + "/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}.nc", + RESDIR + + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}.nc", threads: 1 resources: mem_mb=2000, benchmark: ( - RDIR + RESDIR + "/benchmarks/prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}" ) script: @@ -256,12 +256,12 @@ rule add_export: export_ports="data/export_ports.csv", costs=COSTDIR + "costs_{planning_horizons}.csv", ship_profile="resources/ship_profile_{h2export}TWh.csv", - network=RDIR - + "/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}.nc", + network=RESDIR + + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}.nc", shapes_path="resources/bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", output: - RDIR - + "/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", + RESDIR + + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", script: "scripts/add_export.py" @@ -288,8 +288,8 @@ rule override_respot: network="networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", energy_totals="data/energy_totals_{demand}_{planning_horizons}.csv", output: - RDIR - + "/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_presec.nc", + RESDIR + + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_presec.nc", script: "scripts/override_respot.py" @@ -525,30 +525,30 @@ rule copy_config: rule solve_sector_network: input: overrides="data/override_component_attrs", - # network=RDIR - # + "/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}.nc", - network=RDIR - + "/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", + # network=RESDIR + # + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}.nc", + network=RESDIR + + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", costs=COSTDIR + "costs_{planning_horizons}.csv", configs=SDIR + "/configs/config.yaml", # included to trigger copy_config rule output: - RDIR + RESDIR + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", shadow: "shallow" log: - solver=RDIR + solver=RESDIR + "/logs/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export_solver.log", - python=RDIR + python=RESDIR + "/logs/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export_python.log", - memory=RDIR + memory=RESDIR + "/logs/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export_memory.log", threads: 25 resources: mem_mb=config["solving"]["mem"], benchmark: ( - RDIR + RESDIR + "/benchmarks/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export" ) script: @@ -568,7 +568,7 @@ rule make_sector_summary: input: overrides="data/override_component_attrs", networks=expand( - RDIR + RESDIR + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", **config["scenario"], **config["costs"], @@ -576,7 +576,7 @@ rule make_sector_summary: ), costs=COSTDIR + "costs_{planning_horizons}.csv", plots=expand( - RDIR + RESDIR + "/maps/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}-costs-all_{planning_horizons}_{discountrate}_{demand}_{h2export}export.pdf", **config["scenario"], **config["costs"], @@ -610,17 +610,17 @@ rule make_sector_summary: rule plot_sector_network: input: overrides="data/override_component_attrs", - network=RDIR + network=RESDIR + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", output: - map=RDIR + map=RESDIR + "/maps/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}-costs-all_{planning_horizons}_{discountrate}_{demand}_{h2export}export.pdf", threads: 2 resources: mem_mb=10000, benchmark: ( - RDIR + RESDIR + "/benchmarks/plot_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export" ) script: @@ -656,17 +656,17 @@ rule prepare_db: params: tech_colors=config["plotting"]["tech_colors"], input: - network=RDIR + network=RESDIR + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", output: - db=RDIR + db=RESDIR + "/summaries/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}-costs-all_{planning_horizons}_{discountrate}_{demand}_{h2export}export.csv", threads: 2 resources: mem_mb=10000, benchmark: ( - RDIR + RESDIR + "/benchmarks/prepare_db/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export" ) script: diff --git a/config.default.yaml b/config.default.yaml index 4e54b0fe..9ad152ad 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -8,7 +8,7 @@ costs_dir: data/ #TODO change to the equivalent of technology data run: name: test_run shared_cutouts: true - + foresight: overnight # option to disable the subworkflow to ease the analyses From 7b55745b223897e435c14eaf17cc7eeec314c426 Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 8 Aug 2024 15:45:13 +0200 Subject: [PATCH 03/16] adjust modular solver options --- Snakefile | 236 +++++++++++------------- config.default.yaml | 379 +++++++++++++++++++++++---------------- scripts/copy_config.py | 15 +- scripts/solve_network.py | 10 +- test/config.test1.yaml | 10 +- 5 files changed, 344 insertions(+), 306 deletions(-) diff --git a/Snakefile b/Snakefile index e48f7be4..e23e6175 100644 --- a/Snakefile +++ b/Snakefile @@ -18,7 +18,7 @@ if not exists("config.yaml"): copyfile("config.default.yaml", "config.yaml") -configfile: PYPSAEARTH_FOLDER + "config.pypsa-earth.yaml" +configfile: "config.pypsa-earth.yaml" configfile: PYPSAEARTH_FOLDER + "/configs/bundle_config.yaml" configfile: PYPSAEARTH_FOLDER + "/configs/powerplantmatching_config.yaml" configfile: "config.yaml" @@ -31,16 +31,19 @@ config.update({"submodule_commit": get_last_commit_message(PYPSAEARTH_FOLDER)}) config["ROOT_PATH"] = os.getcwd() run = config.get("run", {}) -SDIR = config["summary_dir"] + run["name"] + "/" if run.get("name") else "" -RESDIR = config["results_dir"] + run["name"] + "/" if run.get("name") else "" +RDIR = run["name"] + "/" if run.get("name") else "" +CDIR = RDIR if not run.get("shared_cutouts") else "" +SDIR = config["summary_dir"] + (run["name"] + "/" if run.get("name") else "") +RESDIR = config["results_dir"] + (run["name"] + "/" if run.get("name") else "") COSTDIR = config["costs_dir"] -RDIR_PE = run["name_subworkflow"] + "/" if run.get("name_subworkflow") else "" -CDIR_PE = RDIR_PE if not run.get("shared_cutouts") else "" + +# RDIR_PE = run["name_subworkflow"] + "/" if run.get("name_subworkflow") else "" +# CDIR_PE = RDIR_PE if not run.get("shared_cutouts") else "" CUTOUTS_PATH = ( "cutouts/" - + CDIR_PE + + CDIR + ("cutout-2013-era5-tutorial.nc" if config["tutorial"] else "cutout-2013-era5.nc") ) @@ -65,11 +68,12 @@ if not config.get("disable_subworkflow", False): config: config - use rule * from pypsaearth data_dir = Path(PYPSAEARTH_FOLDER) / "data" + + rule get_data: output: [ @@ -107,7 +111,7 @@ rule prepare_sector_networks: RESDIR + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}.nc", **config["scenario"], - **config["costs"] + **config["costs"], ), @@ -118,7 +122,7 @@ rule override_res_all_nets: + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_presec.nc", **config["scenario"], **config["costs"], - **config["export"] + **config["export"], ), @@ -129,7 +133,7 @@ rule solve_sector_networks: + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", **config["scenario"], **config["costs"], - **config["export"] + **config["export"], ), @@ -182,8 +186,8 @@ if not config["custom_data"]["gas_network"]: custom_gas_network=config["custom_data"]["gas_network"], input: regions_onshore="resources/" - + RDIR_PE - + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", + + RDIR + + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", output: clustered_gas_network="resources/gas_networks/gas_network_elec_s{simpl}_{clusters}.csv", # TODO: Should be a own snakemake rule @@ -217,11 +221,13 @@ rule prepare_sector_network: district_heat_share="resources/demand/heat/district_heat_share_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", biomass_transport_costs="data/temp_hard_coded/biomass_transport_costs.csv", shapes_path="resources/" - + RDIR_PE - + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", - pipelines="data_custom/pipelines.csv" - if config["custom_data"]["gas_network"] - else "resources/gas_networks/gas_network_elec_s{simpl}_{clusters}.csv", + + RDIR + + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", + pipelines=( + "data_custom/pipelines.csv" + if config["custom_data"]["gas_network"] + else "resources/gas_networks/gas_network_elec_s{simpl}_{clusters}.csv" + ), output: RESDIR + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}.nc", @@ -265,8 +271,8 @@ rule add_export: network=RESDIR + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}.nc", shapes_path="resources/" - + RDIR_PE - + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", + + RDIR + + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", output: RESDIR + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", @@ -293,9 +299,7 @@ rule override_respot: for planning_horizons in config["scenario"]["planning_horizons"] }, overrides="data/override_component_attrs", - network= - "networks/" + RDIR_PE + "elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" - , + network="networks/" + RDIR + "elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", energy_totals="data/energy_totals_{demand}_{planning_horizons}.csv", output: RESDIR @@ -306,7 +310,7 @@ rule override_respot: rule prepare_transport_data: input: - network="networks/" + RDIR_PE + "elec_s{simpl}_{clusters}.nc", + network="networks/" + RDIR + "elec_s{simpl}_{clusters}.nc", energy_totals_name="data/energy_totals_{demand}_{planning_horizons}.csv", traffic_data_KFZ="data/emobility/KFZ__count", traffic_data_Pkw="data/emobility/Pkw__count", @@ -350,7 +354,7 @@ rule build_cop_profiles: rule prepare_heat_data: input: - network="networks/" + RDIR_PE + "elec_s{simpl}_{clusters}.nc", + network="networks/" + RDIR + "elec_s{simpl}_{clusters}.nc", energy_totals_name="data/energy_totals_{demand}_{planning_horizons}.csv", clustered_pop_layout="resources/population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", temp_air_total="resources/temperatures/temp_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", @@ -410,8 +414,8 @@ rule build_solar_thermal_profiles: pop_layout_urban="resources/population_shares/pop_layout_urban_{planning_horizons}.nc", pop_layout_rural="resources/population_shares/pop_layout_rural_{planning_horizons}.nc", regions_onshore="resources/" - + RDIR_PE - + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", + + RDIR + + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", cutout=CUTOUTS_PATH, output: solar_thermal_total="resources/demand/heat/solar_thermal_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", @@ -429,7 +433,7 @@ rule build_population_layouts: params: planning_horizons=config["scenario"]["planning_horizons"][0], input: - nuts3_shapes="resources/" + RDIR_PE + "shapes/gadm_shapes.geojson", + nuts3_shapes="resources/" + RDIR + "shapes/gadm_shapes.geojson", urban_percent="data/urban_percent.csv", cutout=CUTOUTS_PATH, output: @@ -462,8 +466,8 @@ rule build_clustered_population_layouts: pop_layout_rural="resources/population_shares/pop_layout_rural_{planning_horizons}.nc", gdp_layout="resources/gdp_shares/gdp_layout_{planning_horizons}.nc", regions_onshore="resources/" - + RDIR_PE - + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", + + RDIR + + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", cutout=CUTOUTS_PATH, output: clustered_pop_layout="resources/population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", @@ -484,8 +488,8 @@ rule build_heat_demand: pop_layout_urban="resources/population_shares/pop_layout_urban_{planning_horizons}.nc", pop_layout_rural="resources/population_shares/pop_layout_rural_{planning_horizons}.nc", regions_onshore="resources/" - + RDIR_PE - + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", + + RDIR + + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", cutout=CUTOUTS_PATH, output: heat_demand_urban="resources/demand/heat/heat_demand_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", @@ -507,8 +511,8 @@ rule build_temperature_profiles: pop_layout_urban="resources/population_shares/pop_layout_urban_{planning_horizons}.nc", pop_layout_rural="resources/population_shares/pop_layout_rural_{planning_horizons}.nc", regions_onshore="resources/" - + RDIR_PE - + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", + + RDIR + + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", cutout=CUTOUTS_PATH, output: temp_soil_total="resources/temperatures/temp_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", @@ -528,9 +532,10 @@ rule build_temperature_profiles: rule copy_config: params: summary_dir=config["summary_dir"], - run=run["name"], + run=run, output: - SDIR + "/configs/config.yaml", + folder=directory(SDIR + "/configs"), + config=SDIR + "/configs/config.yaml", threads: 1 resources: mem_mb=1000, @@ -542,37 +547,37 @@ rule copy_config: if config["foresight"] == "overnight": -rule solve_sector_network: - input: - overrides="data/override_component_attrs", - # network=RESDIR - # + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}.nc", - network=RESDIR - + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", - costs=COSTDIR + "costs_{planning_horizons}.csv", - configs=SDIR + "/configs/config.yaml", # included to trigger copy_config rule - output: - RESDIR - + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", - shadow: - "shallow" - log: - solver=RESDIR - + "/logs/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export_solver.log", - python=RESDIR - + "/logs/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export_python.log", - memory=RESDIR - + "/logs/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export_memory.log", - threads: 25 - resources: - mem_mb=config["solving"]["mem"], - benchmark: - ( + rule solve_sector_network: + input: + overrides="data/override_component_attrs", + # network=RESDIR + # + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}.nc", + network=RESDIR + + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", + costs=COSTDIR + "costs_{planning_horizons}.csv", + configs=SDIR + "/configs/config.yaml", # included to trigger copy_config rule + output: RESDIR - + "/benchmarks/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export" - ) - script: - "scripts/solve_network.py" + + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", + shadow: + "shallow" + log: + solver=RESDIR + + "/logs/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export_solver.log", + python=RESDIR + + "/logs/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export_python.log", + memory=RESDIR + + "/logs/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export_memory.log", + threads: 25 + resources: + mem_mb=config["solving"]["mem"], + benchmark: + ( + RESDIR + + "/benchmarks/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export" + ) + script: + "scripts/solve_network.py" rule make_sector_summary: @@ -592,7 +597,7 @@ rule make_sector_summary: + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", **config["scenario"], **config["costs"], - **config["export"] + **config["export"], ), costs=COSTDIR + "costs_{planning_horizons}.csv", plots=expand( @@ -600,7 +605,7 @@ rule make_sector_summary: + "/maps/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}-costs-all_{planning_horizons}_{discountrate}_{demand}_{h2export}export.pdf", **config["scenario"], **config["costs"], - **config["export"] + **config["export"], ), output: nodal_costs=SDIR + "/csvs/nodal_costs.csv", @@ -693,31 +698,6 @@ rule prepare_db: "scripts/prepare_db.py" -rule run_test: - params: - dummy="This is a dummy parameter to satisfy snakefmt", - run: - import yaml - - with open(PYPSAEARTH_FOLDER + "/config.tutorial.yaml") as file: - config_pypsaearth = yaml.full_load(file) - config_pypsaearth["retrieve_databundle"] = {"show_progress": False} - config_pypsaearth["electricity"]["extendable_carriers"]["Store"] = [] - config_pypsaearth["electricity"]["extendable_carriers"]["Link"] = [] - config_pypsaearth["electricity"]["co2limit"] = 7.75e7 - config_pypsaearth["run"] = { - "name": "tutorial", - "shared_cutouts": True, - } - - with open("./config.pypsa-earth.yaml", "w") as wfile: - yaml.dump(config_pypsaearth, wfile) - - shell("cp test/config.test1.yaml config.yaml") - shell("snakemake --cores all solve_all_networks --forceall") - - - rule build_industrial_distribution_key: #default data params: countries=config["countries"], @@ -726,14 +706,14 @@ rule build_industrial_distribution_key: #default data industry_database=config["custom_data"]["industry_database"], input: regions_onshore="resources/" - + RDIR_PE - + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", + + RDIR + + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", clustered_pop_layout="resources/population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", clustered_gdp_layout="resources/gdp_shares/gdp_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", industrial_database="data/industrial_database.csv", shapes_path="resources/" - + RDIR_PE - + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", + + RDIR + + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", output: industrial_distribution_key="resources/demand/industrial_distribution_key_elec_s{simpl}_{clusters}_{planning_horizons}.csv", threads: 1 @@ -807,10 +787,10 @@ rule build_existing_heating_distribution: resources: mem_mb=2000, log: - RDIR + RESDIR + "/logs/build_existing_heating_distribution_{demand}_s{simpl}_{clusters}_{planning_horizons}.log", benchmark: - RDIR + RESDIR +"/benchmarks/build_existing_heating_distribution/{demand}_s{simpl}_{clusters}_{planning_horizons}" script: "scripts/build_existing_heating_distribution.py" @@ -825,16 +805,12 @@ if config["foresight"] == "myopic": existing_capacities=config["existing_capacities"], costs=config["costs"], input: - network=RDIR + network=RESDIR + "/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", - powerplants=pypsaearth("resources/" + RDIR_PE + "powerplants.csv"), - busmap_s=pypsaearth( - "resources/" + RDIR_PE + "bus_regions/busmap_elec_s{simpl}.csv" - ), + powerplants="resources/" + RDIR + "powerplants.csv", + busmap_s="resources/" + RDIR + "bus_regions/busmap_elec_s{simpl}.csv", busmap=pypsaearth( - "resources/" - + RDIR_PE - + "bus_regions/busmap_elec_s{simpl}_{clusters}.csv" + "resources/" + RDIR + "bus_regions/busmap_elec_s{simpl}_{clusters}.csv" ), clustered_pop_layout="resources/population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", costs=CDIR @@ -843,7 +819,7 @@ if config["foresight"] == "myopic": cop_air_total="resources/cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", existing_heating_distribution="resources/heating/existing_heating_distribution_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", output: - RDIR + RESDIR + "/prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", wildcard_constraints: # TODO: The first planning_horizon needs to be aligned across scenarios @@ -854,19 +830,19 @@ if config["foresight"] == "myopic": resources: mem_mb=2000, log: - RDIR + RESDIR + "/logs/add_existing_baseyear_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.log", benchmark: - RDIR + RESDIR +"/benchmarks/add_existing_baseyear/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export" script: "scripts/add_existing_baseyear.py" def input_profile_tech_brownfield(w): return { - f"profile_{tech}": pypsaearth( - f"resources/" + RDIR_PE + "renewable_profiles/profile_{tech}.nc" - ) + f"profile_{tech}": f"resources/" + + RDIR + + "renewable_profiles/profile_{tech}.nc" for tech in config["electricity"]["renewable_carriers"] if tech != "hydro" } @@ -895,32 +871,28 @@ if config["foresight"] == "myopic": carriers=config["electricity"]["renewable_carriers"], input: # unpack(input_profile_tech_brownfield), - simplify_busmap=pypsaearth( - "resources/" + RDIR_PE + "bus_regions/busmap_elec_s{simpl}.csv" - ), - cluster_busmap=pypsaearth( - "resources/" - + RDIR_PE - + "bus_regions/busmap_elec_s{simpl}_{clusters}.csv" - ), - network=RDIR + simplify_busmap="resources/" + RDIR + "bus_regions/busmap_elec_s{simpl}.csv", + cluster_busmap="resources/" + + RDIR + + "bus_regions/busmap_elec_s{simpl}_{clusters}.csv", + network=RESDIR + "/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", network_p=solved_previous_horizon, #solved network at previous time step costs=CDIR + "costs_{planning_horizons}.csv", cop_soil_total="resources/cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", cop_air_total="resources/cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", output: - RDIR + RESDIR + "/prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", threads: 4 resources: mem_mb=10000, log: - RDIR + RESDIR + "/logs/add_brownfield_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.log", benchmark: ( - RDIR + RESDIR + "/benchmarks/add_brownfield/elec_s{simpl}_ec_{clusters}_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export" ) script: @@ -938,30 +910,30 @@ if config["foresight"] == "myopic": ), input: overrides="data/override_component_attrs", - network=RDIR + network=RESDIR + "/prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", costs=CDIR + "costs_{planning_horizons}.csv", configs=SDIR + "/configs/config.yaml", # included to trigger copy_config rule output: - network=RDIR + network=RESDIR + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", - # config=RDIR + # config=RESDIR # + "/configs/config.elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.yaml", shadow: "shallow" log: - solver=RDIR + solver=RESDIR + "/logs/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export_solver.log", - python=RDIR + python=RESDIR + "/logs/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export_python.log", - memory=RDIR + memory=RESDIR + "/logs/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export_memory.log", threads: 25 resources: mem_mb=config["solving"]["mem"], benchmark: ( - RDIR + RESDIR + "/benchmarks/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export" ) script: @@ -970,7 +942,7 @@ if config["foresight"] == "myopic": rule solve_all_networks_myopic: input: networks=expand( - RDIR + RESDIR + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", **config["scenario"], **config["costs"], diff --git a/config.default.yaml b/config.default.yaml index 3b89cda9..012a5bd4 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -3,13 +3,13 @@ tutorial: false results_dir: results/ summary_dir: results/ -costs_dir: data/ #TODO change to the equivalent of technology data +costs_dir: data/ # TODO change to the equivalent of technology data run: name: "test_run" # use this to keep track of runs with different settings - name_subworkflow: "" # scenario name of the pypsa-earth subworkflow - shared_cutouts: true # set to true to share the default cutout(s) across runs - # Note: value false requires build_cutout to be enabled + name_subworkflow: "" # scenario name of the pypsa-earth subworkflow + shared_cutouts: true # set to true to share the default cutout(s) across runs + # Note: value false requires build_cutout to be enabled foresight: overnight @@ -18,27 +18,27 @@ disable_subworkflow: false scenario: simpl: # only relevant for PyPSA-Eur - - "" + - "" clusters: # number of nodes in Europe, any integer between 37 (1 node per country-zone) and several hundred - - 10 + - 10 planning_horizons: # investment years for myopic and perfect; or costs year for overnight - - 2030 + - 2030 ll: - - "c1.0" + - "c1.0" opts: - - "Co2L" + - "Co2L" sopts: - - "144H" + - "144H" demand: - - "AB" + - "AB" -electricity: +electricity: extendable_carriers: Link: [] policy_config: hydrogen: - temporal_matching: "no_res_matching" #either "h2_yearly_matching", "h2_monthly_matching", "no_res_matching" + temporal_matching: "no_res_matching" # either "h2_yearly_matching", "h2_monthly_matching", "no_res_matching" spatial_matching: false additionality: false # RE electricity is equal to the amount required for additional hydrogen export compared to the 0 export case ("reference_case") allowed_excess: 1.0 @@ -50,7 +50,7 @@ policy_config: clustering_options: alternative_clustering: true -countries: ['MA'] +countries: ["MA"] demand_data: update_data: true # if true, the workflow downloads the energy balances data saved in data/demand/unsd/data again. Turn on for the first run. @@ -59,15 +59,13 @@ demand_data: other_industries: false # Whether or not to include industries that are not specified. some countries have has exaggerated numbers, check carefully. aluminium_year: 2019 # Year of the aluminium demand data specified in `data/AL_production.csv` - enable: retrieve_cost_data: true # if true, the workflow overwrites the cost data saved in data/costs again - retrieve_irena: true #If true, downloads the IRENA data + retrieve_irena: true #If true, downloads the IRENA data fossil_reserves: oil: 100 #TWh Maybe redundant - export: h2export: [10] # Yearly export demand in TWh store: true # [True, False] # specifies whether an export store to balance demand is implemented @@ -92,7 +90,6 @@ custom_data: custom_sectors: false gas_network: false # If "True" then a custom .csv file must be placed in "resources/custom_data/pipelines.csv" , If "False" the user can choose btw "greenfield" or Model built-in datasets. Please refer to ["sector"] below. - costs: # Costs used in PyPSA-Earth-Sec. Year depends on the wildcard planning_horizon in the scenario section version: v0.6.2 lifetime: 25 #default lifetime @@ -118,7 +115,6 @@ costs: # Costs used in PyPSA-Earth-Sec. Year depends on the wildcard planning_ho lines: length_factor: 1.25 #to estimate offwind connection costs - industry: reference_year: 2015 @@ -129,22 +125,40 @@ solar_thermal: azimuth: 180. existing_capacities: - grouping_years_power: [1960, 1965, 1970, 1975, 1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2020, 2025, 2030] + grouping_years_power: + [ + 1960, + 1965, + 1970, + 1975, + 1980, + 1985, + 1990, + 1995, + 2000, + 2005, + 2010, + 2015, + 2020, + 2025, + 2030, + ] grouping_years_heat: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2019] # these should not extend 2020 threshold_capacity: 10 default_heating_lifetime: 20 conventional_carriers: - - lignite - - coal - - oil - - uranium + - lignite + - coal + - oil + - uranium sector: gas: spatial_gas: true # ALWAYS TRUE network: false # ALWAYS FALSE for now (NOT USED) network_data: GGIT # Global dataset -> 'GGIT' , European dataset -> 'IGGIELGN' - network_data_GGIT_status: ['Construction', 'Operating', 'Idle', 'Shelved', 'Mothballed', 'Proposed'] + network_data_GGIT_status: + ["Construction", "Operating", "Idle", "Shelved", "Mothballed", "Proposed"] hydrogen: network: true H2_retrofit_capacity_per_CH4: 0.6 @@ -163,27 +177,29 @@ sector: spatial_oil: true district_heating: - potential: 0.3 #maximum fraction of urban demand which can be supplied by district heating + potential: + 0.3 #maximum fraction of urban demand which can be supplied by district heating #increase of today's district heating demand to potential maximum district heating share #progress = 0 means today's district heating share, progress=-1 means maximum fraction of urban demand is supplied by district heating - progress: 1 + progress: + 1 #2020: 0.0 #2030: 0.3 #2040: 0.6 #2050: 1.0 district_heating_loss: 0.15 - reduce_space_heat_exogenously: true # reduces space heat demand by a given factor (applied before losses in DH) + reduce_space_heat_exogenously: true # reduces space heat demand by a given factor (applied before losses in DH) # this can represent e.g. building renovation, building demolition, or if # the factor is negative: increasing floor area, increased thermal comfort, population growth reduce_space_heat_exogenously_factor: 0.29 # per unit reduction in space heat demand # the default factors are determined by the LTS scenario from http://tool.european-calculator.eu/app/buildings/building-types-area/?levers=1ddd4444421213bdbbbddd44444ffffff11f411111221111211l212221 - # 2020: 0.10 # this results in a space heat demand reduction of 10% - # 2025: 0.09 # first heat demand increases compared to 2020 because of larger floor area per capita - # 2030: 0.09 - # 2035: 0.11 - # 2040: 0.16 - # 2045: 0.21 - # 2050: 0.29 + # 2020: 0.10 # this results in a space heat demand reduction of 10% + # 2025: 0.09 # first heat demand increases compared to 2020 because of larger floor area per capita + # 2030: 0.09 + # 2035: 0.11 + # 2040: 0.16 + # 2045: 0.21 + # 2050: 0.29 tes: true tes_tau: # 180 day time constant for centralised, 3 day for decentralised @@ -217,7 +233,7 @@ sector: transport_internal_combustion_efficiency: 0.3 industry_util_factor: 0.7 - biomass_transport: true # biomass transport between nodes + biomass_transport: true # biomass transport between nodes biomass_transport_default_cost: 0.1 #EUR/km/MWh solid_biomass_potential: 40 # TWh/a, Potential of whole modelled area biogas_potential: 0.5 # TWh/a, Potential of whole modelled area @@ -324,32 +340,79 @@ snapshots: # atlite: # cutout: ./cutouts/africa-2013-era5.nc -build_osm_network: # TODO: To Remove this once we merge pypsa-earth and pypsa-earth-sec - force_ac: false # When true, it forces all components (lines and substation) to be AC-only. To be used if DC assets create problem. +build_osm_network: # TODO: To Remove this once we merge pypsa-earth and pypsa-earth-sec + force_ac: false # When true, it forces all components (lines and substation) to be AC-only. To be used if DC assets create problem. solving: - #tmpdir: "path/to/tmp" options: formulation: kirchhoff - clip_p_max_pu: 1.e-2 - load_shedding: false + load_shedding: true noisy_costs: true - skip_iterations: true - track_iterations: false min_iterations: 4 max_iterations: 6 + clip_p_max_pu: 0.01 + skip_iterations: true + track_iterations: false + # nhours: 10 solver: name: gurobi - threads: 25 - method: 2 # barrier - crossover: 0 - BarConvTol: 1.e-6 - Seed: 123 - AggFill: 0 - PreDual: 0 - GURO_PAR_BARDENSETHRESH: 200 - #FeasibilityTol: 1.e-6 + options: gurobi-default + + solver_options: + highs-default: + # refer to https://ergo-code.github.io/HiGHS/dev/options/definitions/ + threads: 4 + solver: "ipm" + run_crossover: "off" + small_matrix_value: 1e-6 + large_matrix_value: 1e9 + primal_feasibility_tolerance: 1e-5 + dual_feasibility_tolerance: 1e-5 + ipm_optimality_tolerance: 1e-4 + parallel: "on" + random_seed: 123 + gurobi-default: + threads: 4 + method: 2 # barrier + crossover: 0 + BarConvTol: 1.e-6 + Seed: 123 + AggFill: 0 + PreDual: 0 + GURO_PAR_BARDENSETHRESH: 200 + gurobi-numeric-focus: + NumericFocus: 3 # Favour numeric stability over speed + method: 2 # barrier + crossover: 0 # do not use crossover + BarHomogeneous: 1 # Use homogeneous barrier if standard does not converge + BarConvTol: 1.e-5 + FeasibilityTol: 1.e-4 + OptimalityTol: 1.e-4 + ObjScale: -0.5 + threads: 8 + Seed: 123 + gurobi-fallback: # Use gurobi defaults + crossover: 0 + method: 2 # barrier + BarHomogeneous: 1 # Use homogeneous barrier if standard does not converge + BarConvTol: 1.e-5 + FeasibilityTol: 1.e-5 + OptimalityTol: 1.e-5 + Seed: 123 + threads: 8 + cplex-default: + threads: 4 + lpmethod: 4 # barrier + solutiontype: 2 # non basic solution, ie no crossover + barrier.convergetol: 1.e-5 + feasopt.tolerance: 1.e-6 + copt-default: + Threads: 8 + LpMethod: 2 + Crossover: 0 + cbc-default: {} # Used in CI + glpk-default: {} # Used in CI mem: 30000 #memory in MB; 20 GB enough for 50+B+I+H2; 100 GB for 181+B+I+H2 @@ -365,47 +428,47 @@ plotting: energy_min: -20000 energy_threshold: 15 vre_techs: - - onwind - - offwind-ac - - offwind-dc - - solar - - ror + - onwind + - offwind-ac + - offwind-dc + - solar + - ror renewable_storage_techs: - - PHS - - hydro + - PHS + - hydro conv_techs: - - OCGT - - CCGT - - Nuclear - - Coal + - OCGT + - CCGT + - Nuclear + - Coal storage_techs: - - hydro+PHS - - battery - - H2 + - hydro+PHS + - battery + - H2 load_carriers: - - AC load + - AC load AC_carriers: - - AC line - - AC transformer + - AC line + - AC transformer link_carriers: - - DC line - - Converter AC-DC + - DC line + - Converter AC-DC heat_links: - - heat pump - - resistive heater - - CHP heat - - CHP electric - - gas boiler - - central heat pump - - central resistive heater - - central CHP heat - - central CHP electric - - central gas boiler + - heat pump + - resistive heater + - CHP heat + - CHP electric + - gas boiler + - central heat pump + - central resistive heater + - central CHP heat + - central CHP electric + - central gas boiler heat_generators: - - gas boiler - - central gas boiler - - solar thermal collector - - central solar thermal collector + - gas boiler + - central gas boiler + - solar thermal collector + - central solar thermal collector tech_colors: SMR CC: "darkblue" gas for industry CC: "brown" @@ -419,28 +482,28 @@ plotting: offshore wind (AC): "#6895dd" offwind-dc: "#74c6f2" offshore wind (DC): "#74c6f2" - wave: '#004444' - hydro: '#3B5323' - hydro reservoir: '#3B5323' - ror: '#78AB46' - run of river: '#78AB46' - hydroelectricity: 'blue' + wave: "#004444" + hydro: "#3B5323" + hydro reservoir: "#3B5323" + ror: "#78AB46" + run of river: "#78AB46" + hydroelectricity: "blue" solar: "orange" solar PV: "#f9d002" solar thermal: coral - solar rooftop: '#ffef60' + solar rooftop: "#ffef60" OCGT: wheat OCGT marginal: sandybrown - OCGT-heat: '#ee8340' - gas boiler: '#ee8340' - gas boilers: '#ee8340' - gas boiler marginal: '#ee8340' - gas-to-power/heat: 'brown' + OCGT-heat: "#ee8340" + gas boiler: "#ee8340" + gas boilers: "#ee8340" + gas boiler marginal: "#ee8340" + gas-to-power/heat: "brown" gas: brown natural gas: brown - SMR: '#4F4F2F' - oil: '#B5A642' - oil boiler: '#B5A677' + SMR: "#4F4F2F" + oil: "#B5A642" + oil boiler: "#B5A677" lines: k transmission lines: k H2: m @@ -448,8 +511,8 @@ plotting: hydrogen storage: m battery: slategray battery storage: slategray - home battery: '#614700' - home battery storage: '#614700' + home battery: "#614700" + home battery storage: "#614700" Nuclear: r Nuclear marginal: r nuclear: r @@ -460,42 +523,42 @@ plotting: Lignite: grey lignite: grey Lignite marginal: grey - CCGT: '#ee8340' - CCGT marginal: '#ee8340' - heat pumps: '#76EE00' - heat pump: '#76EE00' - air heat pump: '#76EE00' - ground heat pump: '#40AA00' - power-to-heat: 'red' + CCGT: "#ee8340" + CCGT marginal: "#ee8340" + heat pumps: "#76EE00" + heat pump: "#76EE00" + air heat pump: "#76EE00" + ground heat pump: "#40AA00" + power-to-heat: "red" resistive heater: pink - Sabatier: '#FF1493' - methanation: '#FF1493' - power-to-gas: 'purple' - power-to-liquid: 'darkgreen' - helmeth: '#7D0552' - DAC: 'deeppink' - co2 stored: '#123456' - CO2 sequestration: '#123456' + Sabatier: "#FF1493" + methanation: "#FF1493" + power-to-gas: "purple" + power-to-liquid: "darkgreen" + helmeth: "#7D0552" + DAC: "deeppink" + co2 stored: "#123456" + CO2 sequestration: "#123456" CC: k - co2: '#123456' - co2 vent: '#654321' - agriculture heat: '#D07A7A' - agriculture machinery oil: '#1e1e1e' - agriculture machinery oil emissions: '#111111' - agriculture electricity: '#222222' - solid biomass for industry co2 from atmosphere: '#654321' - solid biomass for industry co2 to stored: '#654321' - solid biomass for industry CC: '#654321' - gas for industry co2 to atmosphere: '#654321' - gas for industry co2 to stored: '#654321' - Fischer-Tropsch: '#44DD33' - kerosene for aviation: '#44BB11' - naphtha for industry: '#44FF55' - land transport oil: '#44DD33' - water tanks: '#BBBBBB' - hot water storage: '#BBBBBB' - hot water charging: '#BBBBBB' - hot water discharging: '#999999' + co2: "#123456" + co2 vent: "#654321" + agriculture heat: "#D07A7A" + agriculture machinery oil: "#1e1e1e" + agriculture machinery oil emissions: "#111111" + agriculture electricity: "#222222" + solid biomass for industry co2 from atmosphere: "#654321" + solid biomass for industry co2 to stored: "#654321" + solid biomass for industry CC: "#654321" + gas for industry co2 to atmosphere: "#654321" + gas for industry co2 to stored: "#654321" + Fischer-Tropsch: "#44DD33" + kerosene for aviation: "#44BB11" + naphtha for industry: "#44FF55" + land transport oil: "#44DD33" + water tanks: "#BBBBBB" + hot water storage: "#BBBBBB" + hot water charging: "#BBBBBB" + hot water discharging: "#999999" # CO2 pipeline: '#999999' CHP: r CHP heat: r @@ -505,39 +568,39 @@ plotting: Electric load: b Heat load: r heat: darkred - rural heat: '#880000' - central heat: '#b22222' - decentral heat: '#800000' - low-temperature heat for industry: '#991111' - process heat: '#FF3333' + rural heat: "#880000" + central heat: "#b22222" + decentral heat: "#800000" + low-temperature heat for industry: "#991111" + process heat: "#FF3333" heat demand: darkred electric demand: k Li ion: grey - district heating: '#CC4E5C' + district heating: "#CC4E5C" retrofitting: purple building retrofitting: purple BEV charger: grey V2G: grey land transport EV: grey electricity: k - gas for industry: '#333333' - solid biomass for industry: '#555555' - industry electricity: '#222222' - industry new electricity: '#222222' - process emissions to stored: '#444444' - process emissions to atmosphere: '#888888' - process emissions: '#222222' - oil emissions: '#666666' - industry oil emissions: '#666666' - land transport oil emissions: '#666666' - land transport fuel cell: '#AAAAAA' - biogas: '#800000' - solid biomass: '#DAA520' - today: '#D2691E' - shipping: '#6495ED' + gas for industry: "#333333" + solid biomass for industry: "#555555" + industry electricity: "#222222" + industry new electricity: "#222222" + process emissions to stored: "#444444" + process emissions to atmosphere: "#888888" + process emissions: "#222222" + oil emissions: "#666666" + industry oil emissions: "#666666" + land transport oil emissions: "#666666" + land transport fuel cell: "#AAAAAA" + biogas: "#800000" + solid biomass: "#DAA520" + today: "#D2691E" + shipping: "#6495ED" shipping oil: "#6495ED" shipping oil emissions: "#6495ED" - electricity distribution grid: 'y' + electricity distribution grid: "y" solid biomass transport: green H2 for industry: "#222222" H2 for shipping: "#6495ED" diff --git a/scripts/copy_config.py b/scripts/copy_config.py index 085c5ca0..4848c90a 100644 --- a/scripts/copy_config.py +++ b/scripts/copy_config.py @@ -12,19 +12,10 @@ if __name__ == "__main__": if "snakemake" not in globals(): - from helpers import mock_snakemake, sets_path_to_root - - os.chdir(os.path.dirname(os.path.abspath(__file__))) + from helpers import mock_snakemake snakemake = mock_snakemake("copy_config") - os.chdir(snakemake.config["ROOT_PATH"]) + directory = snakemake.output["folder"] for f, name in files_to_copy.items(): - copy( - f, - snakemake.params.summary_dir - + "/" - + snakemake.params.run["name"] - + "/configs/" - + name, - ) + copy(f, directory + "/" + name) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 251f6f96..c89f59dd 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -486,9 +486,13 @@ def extra_functionality(n, snapshots): def solve_network(n, config, opts="", **kwargs): - solver_options = config["solving"]["solver"].copy() - solver_name = solver_options.pop("name") - cf_solving = config["solving"]["options"] + solving = config["solving"] + set_of_options = solving["solver"]["options"] + cf_solving = solving["options"] + + solver_options = solving["solver_options"][set_of_options] if set_of_options else {} + solver_name = solving["solver"]["name"] + track_iterations = cf_solving.get("track_iterations", False) min_iterations = cf_solving.get("min_iterations", 4) max_iterations = cf_solving.get("max_iterations", 6) diff --git a/test/config.test1.yaml b/test/config.test1.yaml index 44203d1e..85e44631 100644 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -1,6 +1,6 @@ tutorial: true -run: +run: name: test1 name_subworkflow: "tutorial" shared_cutouts: true @@ -13,6 +13,14 @@ scenario: countries: ['NG', 'BJ'] + +electricity: + extendable_carriers: + Store: [] + Link: [] + + co2limit: 7.75e7 + export: h2export: [120] # Yearly export demand in TWh store: true # [True, False] # specifies whether an export store to balance demand is implemented From 52d77d034f8d19109f924f0aad74da5c5362824b Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 8 Aug 2024 16:55:39 +0200 Subject: [PATCH 04/16] resolve global paths --- Snakefile | 126 ++++++++++++++++++++--------------------- test/config.test1.yaml | 1 - 2 files changed, 62 insertions(+), 65 deletions(-) diff --git a/Snakefile b/Snakefile index e23e6175..b76fd573 100644 --- a/Snakefile +++ b/Snakefile @@ -30,16 +30,14 @@ config.update({"git_commit": get_last_commit_message(".")}) config.update({"submodule_commit": get_last_commit_message(PYPSAEARTH_FOLDER)}) config["ROOT_PATH"] = os.getcwd() + run = config.get("run", {}) RDIR = run["name"] + "/" if run.get("name") else "" CDIR = RDIR if not run.get("shared_cutouts") else "" -SDIR = config["summary_dir"] + (run["name"] + "/" if run.get("name") else "") -RESDIR = config["results_dir"] + (run["name"] + "/" if run.get("name") else "") +SDIR = str(Path(config["summary_dir"]) / RDIR) # use Path to flexibly allow closing "/" +RESDIR = str(Path(config["results_dir"]) / RDIR) COSTDIR = config["costs_dir"] -# RDIR_PE = run["name_subworkflow"] + "/" if run.get("name_subworkflow") else "" -# CDIR_PE = RDIR_PE if not run.get("shared_cutouts") else "" - CUTOUTS_PATH = ( "cutouts/" @@ -130,7 +128,7 @@ rule solve_sector_networks: input: expand( RESDIR - + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", + + "postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", **config["scenario"], **config["costs"], **config["export"], @@ -237,7 +235,7 @@ rule prepare_sector_network: benchmark: ( RESDIR - + "/benchmarks/prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}" + + "benchmarks/prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}" ) script: "scripts/prepare_sector_network.py" @@ -534,13 +532,13 @@ rule copy_config: summary_dir=config["summary_dir"], run=run, output: - folder=directory(SDIR + "/configs"), - config=SDIR + "/configs/config.yaml", + folder=directory(SDIR + "configs"), + config=SDIR + "configs/config.yaml", threads: 1 resources: mem_mb=1000, benchmark: - SDIR + "/benchmarks/copy_config" + SDIR + "benchmarks/copy_config" script: "scripts/copy_config.py" @@ -558,23 +556,23 @@ if config["foresight"] == "overnight": configs=SDIR + "/configs/config.yaml", # included to trigger copy_config rule output: RESDIR - + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", + + "postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", shadow: "shallow" log: solver=RESDIR - + "/logs/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export_solver.log", + + "logs/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export_solver.log", python=RESDIR - + "/logs/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export_python.log", + + "logs/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export_python.log", memory=RESDIR - + "/logs/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export_memory.log", + + "logs/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export_memory.log", threads: 25 resources: mem_mb=config["solving"]["mem"], benchmark: ( RESDIR - + "/benchmarks/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export" + + "benchmarks/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export" ) script: "scripts/solve_network.py" @@ -594,7 +592,7 @@ rule make_sector_summary: overrides="data/override_component_attrs", networks=expand( RESDIR - + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", + + "postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", **config["scenario"], **config["costs"], **config["export"], @@ -608,26 +606,26 @@ rule make_sector_summary: **config["export"], ), output: - nodal_costs=SDIR + "/csvs/nodal_costs.csv", - nodal_capacities=SDIR + "/csvs/nodal_capacities.csv", - nodal_cfs=SDIR + "/csvs/nodal_cfs.csv", - cfs=SDIR + "/csvs/cfs.csv", - costs=SDIR + "/csvs/costs.csv", - capacities=SDIR + "/csvs/capacities.csv", - curtailment=SDIR + "/csvs/curtailment.csv", - energy=SDIR + "/csvs/energy.csv", - supply=SDIR + "/csvs/supply.csv", - supply_energy=SDIR + "/csvs/supply_energy.csv", - prices=SDIR + "/csvs/prices.csv", - weighted_prices=SDIR + "/csvs/weighted_prices.csv", - market_values=SDIR + "/csvs/market_values.csv", - price_statistics=SDIR + "/csvs/price_statistics.csv", - metrics=SDIR + "/csvs/metrics.csv", + nodal_costs=SDIR + "csvs/nodal_costs.csv", + nodal_capacities=SDIR + "csvs/nodal_capacities.csv", + nodal_cfs=SDIR + "csvs/nodal_cfs.csv", + cfs=SDIR + "csvs/cfs.csv", + costs=SDIR + "csvs/costs.csv", + capacities=SDIR + "csvs/capacities.csv", + curtailment=SDIR + "csvs/curtailment.csv", + energy=SDIR + "csvs/energy.csv", + supply=SDIR + "csvs/supply.csv", + supply_energy=SDIR + "csvs/supply_energy.csv", + prices=SDIR + "csvs/prices.csv", + weighted_prices=SDIR + "csvs/weighted_prices.csv", + market_values=SDIR + "csvs/market_values.csv", + price_statistics=SDIR + "csvs/price_statistics.csv", + metrics=SDIR + "csvs/metrics.csv", threads: 2 resources: mem_mb=10000, benchmark: - SDIR + "/benchmarks/make_summary" + SDIR + "benchmarks/make_summary" script: "scripts/make_summary.py" @@ -636,7 +634,7 @@ rule plot_sector_network: input: overrides="data/override_component_attrs", network=RESDIR - + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", + + "postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", output: map=RESDIR + "/maps/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}-costs-all_{planning_horizons}_{discountrate}_{demand}_{h2export}export.pdf", @@ -646,7 +644,7 @@ rule plot_sector_network: benchmark: ( RESDIR - + "/benchmarks/plot_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export" + + "benchmarks/plot_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export" ) script: "scripts/plot_network.py" @@ -654,18 +652,18 @@ rule plot_sector_network: rule plot_sector_summary: input: - costs=SDIR + "/csvs/costs.csv", - energy=SDIR + "/csvs/energy.csv", - balances=SDIR + "/csvs/supply_energy.csv", + costs=SDIR + "csvs/costs.csv", + energy=SDIR + "csvs/energy.csv", + balances=SDIR + "csvs/supply_energy.csv", output: - costs=SDIR + "/graphs/costs.pdf", - energy=SDIR + "/graphs/energy.pdf", - balances=SDIR + "/graphs/balances-energy.pdf", + costs=SDIR + "graphs/costs.pdf", + energy=SDIR + "graphs/energy.pdf", + balances=SDIR + "graphs/balances-energy.pdf", threads: 2 resources: mem_mb=10000, benchmark: - SDIR + "/benchmarks/plot_summary" + SDIR + "benchmarks/plot_summary" script: "scripts/plot_summary.py" @@ -682,17 +680,17 @@ rule prepare_db: tech_colors=config["plotting"]["tech_colors"], input: network=RESDIR - + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", + + "postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", output: db=RESDIR - + "/summaries/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}-costs-all_{planning_horizons}_{discountrate}_{demand}_{h2export}export.csv", + + "summaries/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}-costs-all_{planning_horizons}_{discountrate}_{demand}_{h2export}export.csv", threads: 2 resources: mem_mb=10000, benchmark: ( RESDIR - + "/benchmarks/prepare_db/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export" + + "benchmarks/prepare_db/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export" ) script: "scripts/prepare_db.py" @@ -788,10 +786,10 @@ rule build_existing_heating_distribution: mem_mb=2000, log: RESDIR - + "/logs/build_existing_heating_distribution_{demand}_s{simpl}_{clusters}_{planning_horizons}.log", + + "logs/build_existing_heating_distribution_{demand}_s{simpl}_{clusters}_{planning_horizons}.log", benchmark: RESDIR - +"/benchmarks/build_existing_heating_distribution/{demand}_s{simpl}_{clusters}_{planning_horizons}" + +"benchmarks/build_existing_heating_distribution/{demand}_s{simpl}_{clusters}_{planning_horizons}" script: "scripts/build_existing_heating_distribution.py" @@ -806,7 +804,7 @@ if config["foresight"] == "myopic": costs=config["costs"], input: network=RESDIR - + "/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", + + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", powerplants="resources/" + RDIR + "powerplants.csv", busmap_s="resources/" + RDIR + "bus_regions/busmap_elec_s{simpl}.csv", busmap=pypsaearth( @@ -820,7 +818,7 @@ if config["foresight"] == "myopic": existing_heating_distribution="resources/heating/existing_heating_distribution_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", output: RESDIR - + "/prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", + + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", wildcard_constraints: # TODO: The first planning_horizon needs to be aligned across scenarios # snakemake does not support passing functions to wildcard_constraints @@ -831,10 +829,10 @@ if config["foresight"] == "myopic": mem_mb=2000, log: RESDIR - + "/logs/add_existing_baseyear_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.log", + + "logs/add_existing_baseyear_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.log", benchmark: RESDIR - +"/benchmarks/add_existing_baseyear/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export" + +"benchmarks/add_existing_baseyear/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export" script: "scripts/add_existing_baseyear.py" @@ -854,7 +852,7 @@ if config["foresight"] == "myopic": return ( RDIR - + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_" + + "postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_" + planning_horizon_p + "_{discountrate}_{demand}_{h2export}export.nc" ) @@ -876,24 +874,24 @@ if config["foresight"] == "myopic": + RDIR + "bus_regions/busmap_elec_s{simpl}_{clusters}.csv", network=RESDIR - + "/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", + + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", network_p=solved_previous_horizon, #solved network at previous time step costs=CDIR + "costs_{planning_horizons}.csv", cop_soil_total="resources/cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", cop_air_total="resources/cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", output: RESDIR - + "/prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", + + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", threads: 4 resources: mem_mb=10000, log: RESDIR - + "/logs/add_brownfield_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.log", + + "logs/add_brownfield_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.log", benchmark: ( RESDIR - + "/benchmarks/add_brownfield/elec_s{simpl}_ec_{clusters}_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export" + + "benchmarks/add_brownfield/elec_s{simpl}_ec_{clusters}_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export" ) script: "./scripts/add_brownfield.py" @@ -911,30 +909,30 @@ if config["foresight"] == "myopic": input: overrides="data/override_component_attrs", network=RESDIR - + "/prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", + + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", costs=CDIR + "costs_{planning_horizons}.csv", - configs=SDIR + "/configs/config.yaml", # included to trigger copy_config rule + configs=SDIR + "configs/config.yaml", # included to trigger copy_config rule output: network=RESDIR - + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", + + "postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", # config=RESDIR - # + "/configs/config.elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.yaml", + # + "configs/config.elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.yaml", shadow: "shallow" log: solver=RESDIR - + "/logs/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export_solver.log", + + "logs/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export_solver.log", python=RESDIR - + "/logs/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export_python.log", + + "logs/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export_python.log", memory=RESDIR - + "/logs/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export_memory.log", + + "logs/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export_memory.log", threads: 25 resources: mem_mb=config["solving"]["mem"], benchmark: ( RESDIR - + "/benchmarks/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export" + + "benchmarks/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export" ) script: "./scripts/solve_network.py" @@ -943,7 +941,7 @@ if config["foresight"] == "myopic": input: networks=expand( RESDIR - + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", + + "postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", **config["scenario"], **config["costs"], **config["export"], diff --git a/test/config.test1.yaml b/test/config.test1.yaml index 85e44631..1864c865 100644 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -2,7 +2,6 @@ tutorial: true run: name: test1 - name_subworkflow: "tutorial" shared_cutouts: true scenario: From f55c84bbdedbd92c1f840a90a439bdba79ab7ca5 Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 8 Aug 2024 16:58:51 +0200 Subject: [PATCH 05/16] update pypsa-earth commit --- pypsa-earth | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypsa-earth b/pypsa-earth index 865cadfc..69817205 160000 --- a/pypsa-earth +++ b/pypsa-earth @@ -1 +1 @@ -Subproject commit 865cadfc5ce01065c9daac8f01f2d4dfa50e2391 +Subproject commit 6981720575a84de451229892fcc194bf02c0a552 From 779482d79a4f6eff28ebdb42edb3c3a1d638e5bc Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 8 Aug 2024 17:48:34 +0200 Subject: [PATCH 06/16] udpate pypsa-earth --- pypsa-earth | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypsa-earth b/pypsa-earth index 69817205..23bc59fe 160000 --- a/pypsa-earth +++ b/pypsa-earth @@ -1 +1 @@ -Subproject commit 6981720575a84de451229892fcc194bf02c0a552 +Subproject commit 23bc59feceed82eee9270111140fba55ec0f915a From 329a4f916fb8eaf26ef637b633b025bd968dd76b Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 8 Aug 2024 18:49:54 +0200 Subject: [PATCH 07/16] consolidate CI yaml --- .github/workflows/ci-linux.yaml | 78 ------------------------------- .github/workflows/ci-mac.yaml | 76 ------------------------------ .github/workflows/ci-windows.yaml | 76 ------------------------------ .github/workflows/ci.yml | 64 +++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 230 deletions(-) delete mode 100644 .github/workflows/ci-linux.yaml delete mode 100644 .github/workflows/ci-mac.yaml delete mode 100644 .github/workflows/ci-windows.yaml create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci-linux.yaml b/.github/workflows/ci-linux.yaml deleted file mode 100644 index 26d59e34..00000000 --- a/.github/workflows/ci-linux.yaml +++ /dev/null @@ -1,78 +0,0 @@ -# SPDX-FileCopyrightText: : 2022 The PyPSA-meets-Africa Authors -# -# SPDX-License-Identifier: CC0-1.0 - -name: CI-linux - -on: - push: - branches: - - main - pull_request: - branches: - - main - schedule: - - cron: "0 5 * * TUE" - -env: - CACHE_NUMBER: 1 # Change this value to manually reset the environment cache - -jobs: - build: - strategy: - fail-fast: false # don't break CI for ubuntu if windows fails before - matrix: - include: - # Matrix required to handle environment caching with Mambaforge - - os: ubuntu-latest - label: ubuntu-latest - prefix: /usr/share/miniconda3/envs/pypsa-earth - - name: ${{ matrix.label }} - runs-on: ${{ matrix.os }} - - defaults: - run: - shell: bash -l {0} - - steps: - - uses: actions/checkout@v2 - - - name: Fill submodule - run: | - git submodule update --init - - - name: Copy environment file - run: | - cp ./pypsa-earth/envs/environment.yaml environment.yaml - - - name: Setup Mambaforge - uses: conda-incubator/setup-miniconda@v2 - with: - miniforge-variant: Mambaforge - miniforge-version: latest - activate-environment: pypsa-earth - use-mamba: true - - - name: Create environment cache - uses: actions/cache@v2 - id: cache - with: - path: ${{ matrix.prefix }} - key: ${{ matrix.label }}--conda-${{ hashFiles('environment.yaml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }} - - - name: Update environment due to outdated or unavailable cache - if: steps.cache.outputs.cache-hit != 'true' - run: mamba env update -n pypsa-earth -f environment.yaml - - - name: Test snakemake workflow - run: | - conda list - make test - - # - name: Test plotting and summaries - # run: | - # snakemake --cores all plot_all_p_nom - # snakemake --cores all plot_all_summaries - # snakemake --cores all make_all_summaries - # rm -rf resources/*.nc resources/*.geojson resources/*.h5 networks results diff --git a/.github/workflows/ci-mac.yaml b/.github/workflows/ci-mac.yaml deleted file mode 100644 index d4c96d26..00000000 --- a/.github/workflows/ci-mac.yaml +++ /dev/null @@ -1,76 +0,0 @@ -# SPDX-FileCopyrightText: : 2022 The PyPSA-meets-Africa Authors -# -# SPDX-License-Identifier: CC0-1.0 - -name: CI-mac - -on: - push: - branches: - - main - pull_request: - branches: - - main - schedule: - - cron: "0 5 * * TUE" - -env: - CACHE_NUMBER: 1 # Change this value to manually reset the environment cache - -jobs: - build: - strategy: - matrix: - include: - - os: macos-latest - label: macos-latest - prefix: /Users/runner/miniconda3/envs/pypsa-earth - - name: ${{ matrix.label }} - runs-on: ${{ matrix.os }} - - defaults: - run: - shell: bash -l {0} - - steps: - - uses: actions/checkout@v2 - - - name: Fill submodule - run: | - git submodule update --init - - - name: Copy environment file - run: | - cp ./pypsa-earth/envs/environment.yaml environment.yaml - - - name: Setup Mambaforge - uses: conda-incubator/setup-miniconda@v2 - with: - miniforge-variant: Mambaforge - miniforge-version: latest - activate-environment: pypsa-earth - use-mamba: true - - - name: Create environment cache - uses: actions/cache@v2 - id: cache - with: - path: ${{ matrix.prefix }} - key: ${{ matrix.label }}-conda-${{ hashFiles('environment.yaml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }} - - - name: Update environment due to outdated or unavailable cache - if: steps.cache.outputs.cache-hit != 'true' - run: mamba env update -n pypsa-earth -f environment.yaml - - - name: Test snakemake workflow - run: | - conda list - make test - - # - name: Test plotting and summaries - # run: | - # snakemake --cores all plot_all_p_nom - # snakemake --cores all plot_all_summaries - # snakemake --cores all make_all_summaries - # rm -rf resources/*.nc resources/*.geojson resources/*.h5 networks results diff --git a/.github/workflows/ci-windows.yaml b/.github/workflows/ci-windows.yaml deleted file mode 100644 index 15de3c36..00000000 --- a/.github/workflows/ci-windows.yaml +++ /dev/null @@ -1,76 +0,0 @@ -# SPDX-FileCopyrightText: : 2022 The PyPSA-meets-Africa Authors -# -# SPDX-License-Identifier: CC0-1.0 - -name: CI-windows - -on: - push: - branches: - - main - pull_request: - branches: - - main - schedule: - - cron: "0 5 * * TUE" - -env: - CACHE_NUMBER: 1 # Change this value to manually reset the environment cache - -jobs: - build: - strategy: - matrix: - include: - - os: windows-latest - label: windows-latest - prefix: C:\Miniconda3\envs\pypsa-earth - - name: ${{ matrix.label }} - runs-on: ${{ matrix.os }} - - defaults: - run: - shell: bash -l {0} - - steps: - - uses: actions/checkout@v2 - - - name: Fill submodule - run: | - git submodule update --init - - - name: Copy environment file - run: | - cp ./pypsa-earth/envs/environment.yaml environment.yaml - - - name: Setup Mambaforge - uses: conda-incubator/setup-miniconda@v2 - with: - miniforge-variant: Mambaforge - miniforge-version: latest - activate-environment: pypsa-earth - use-mamba: true - - - name: Create environment cache - uses: actions/cache@v2 - id: cache - with: - path: ${{ matrix.prefix }} - key: ${{ matrix.label }}-conda-${{ hashFiles('environment.yaml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }} - - - name: Update environment due to outdated or unavailable cache - if: steps.cache.outputs.cache-hit != 'true' - run: mamba env update -n pypsa-earth -f environment.yaml - - - name: Test snakemake workflow - run: | - conda list - make test - - # - name: Test plotting and summaries - # run: | - # snakemake --cores all plot_all_p_nom - # snakemake --cores all plot_all_summaries - # snakemake --cores all make_all_summaries - # rm -rf resources/*.nc resources/*.geojson resources/*.h5 networks results diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..57350bb6 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,64 @@ +name: CI + +on: + push: + branches: + - main + pull_request: + branches: + - main + schedule: + - cron: "0 5 * * TUE" + +env: + CACHE_NUMBER: 2 # Change this value to manually reset the environment cache + +jobs: + build: + strategy: + fail-fast: false + max-parallel: 3 + matrix: + os: + - ubuntu-latest + - macos-latest + # - windows-latest + + runs-on: ${{ matrix.os }} + + defaults: + run: + shell: bash -l {0} + + steps: + - uses: actions/checkout@v2 + + + - name: Setup micromamba + uses: mamba-org/setup-micromamba@v1 + with: + micromamba-version: latest + environment-file: pypsa-earth/envs/environment.yaml + log-level: debug + init-shell: bash + cache-environment: true + cache-downloads: true + + - name: Set cache dates + run: | + echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV + + - name: Cache data and cutouts folders + uses: actions/cache@v3 + with: + path: | + data + cutouts + key: data-cutouts-${{ env.WEEK }}-${{ env.CACHE_NUMBER }} + + + - name: Conda list + run: conda list + + - name: Run Test + run: make test From 15a5c95b3e052fa091b11fa5bb4d16d03601e6b7 Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 8 Aug 2024 23:01:07 +0200 Subject: [PATCH 08/16] yamllint: align formatting and config --- .github/workflows/linter.yml | 14 ++++++------- .pre-commit-config.yaml | 2 +- .yamllint | 9 ++++---- config.default.yaml | 12 +++++------ config.pypsa-earth.yaml | 7 +++---- configs/powerplantmatching_config.yaml | 4 ++-- test/config.test_myopic.yaml | 29 +++++++++++++------------- 7 files changed, 37 insertions(+), 40 deletions(-) diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 8ac10577..30aa194f 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -32,18 +32,18 @@ jobs: # Load all steps # ################## steps: - ########################## - # Checkout the code base # - ########################## + ########################## + # Checkout the code base # + ########################## - name: Checkout Code uses: actions/checkout@v2 with: - # Full git history is needed to get a proper list of changed files within `super-linter` + # Full git history is needed to get a proper list of changed files within `super-linter` fetch-depth: 0 - ################################ - # Run Linter against code base # - ################################ + ################################ + # Run Linter against code base # + ################################ - name: Lint Code Base uses: docker://ghcr.io/github/super-linter:slim-v4 env: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 94b00a8f..76664e71 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -55,7 +55,7 @@ repos: rev: v1.33.0 hooks: - id: yamllint - args: [--format, parsable, -c=.yamllint] + args: [-c=.yamllint] # Format Snakemake rule / workflow files - repo: https://github.com/snakemake/snakefmt diff --git a/.yamllint b/.yamllint index a4800876..29dffec6 100644 --- a/.yamllint +++ b/.yamllint @@ -9,14 +9,13 @@ extends: default rules: braces: - # Do not allow flow mappings using curly braces "{" and "}" - forbid: true + forbid: false brackets: max-spaces-inside: 0 max-spaces-inside-empty: 0 comments: - require-starting-space: true - min-spaces-from-content: 2 + require-starting-space: false + min-spaces-from-content: 0 # Force correct indentation of comments # yamllint disable-line rule:braces comments-indentation: {} @@ -35,7 +34,7 @@ rules: key-duplicates: {} line-length: level: warning - max: 88 + max: 350 new-line-at-end-of-file: enable truthy: check-keys: false # Disable truthy check hits on keys like "on": ... diff --git a/config.default.yaml b/config.default.yaml index fa49dbf4..c508e352 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -162,13 +162,13 @@ sector: district_heating: potential: 0.3 #maximum fraction of urban demand which can be supplied by district heating - #increase of today's district heating demand to potential maximum district heating share - #progress = 0 means today's district heating share, progress=-1 means maximum fraction of urban demand is supplied by district heating + #increase of today's district heating demand to potential maximum district heating share + #progress = 0 means today's district heating share, progress=-1 means maximum fraction of urban demand is supplied by district heating progress: 1 - #2020: 0.0 - #2030: 0.3 - #2040: 0.6 - #2050: 1.0 + # 2020: 0.0 + # 2030: 0.3 + # 2040: 0.6 + # 2050: 1.0 district_heating_loss: 0.15 reduce_space_heat_exogenously: true # reduces space heat demand by a given factor (applied before losses in DH) # this can represent e.g. building renovation, building demolition, or if diff --git a/config.pypsa-earth.yaml b/config.pypsa-earth.yaml index 91673f1d..235d384e 100644 --- a/config.pypsa-earth.yaml +++ b/config.pypsa-earth.yaml @@ -26,8 +26,7 @@ custom_rules: [] # Default empty [] or link to custom rule file e.g. ["my_folde run: name: "" # use this to keep track of runs with different settings - shared_cutouts: true # set to true to share the default cutout(s) across runs - # Note: value false requires build_cutout to be enabled + shared_cutouts: true # set to true to share the default cutout(s) across runs; Note: value false requires build_cutout to be enabled scenario: simpl: [''] @@ -279,8 +278,8 @@ renewable: method: pv panel: CSi orientation: latitude_optimal # will lead into optimal design - # slope: 0. # slope: 0 represent a flat panel - # azimuth: 180. # azimuth: 180 south orientation + # slope: 0. # slope: 0 represent a flat panel + # azimuth: 180. # azimuth: 180 south orientation capacity_per_sqkm: 4.6 # From 1.7 to 4.6 addresses issue #361 # Determined by comparing uncorrected area-weighted full-load hours to those # published in Supplementary Data to diff --git a/configs/powerplantmatching_config.yaml b/configs/powerplantmatching_config.yaml index 86ac5948..df674848 100644 --- a/configs/powerplantmatching_config.yaml +++ b/configs/powerplantmatching_config.yaml @@ -14,7 +14,7 @@ google_api_key: #matching config matching_sources: - # - CARMA # deprecated as no more public +#- CARMA # deprecated as no more public - GEO - GPD - GBPT @@ -30,7 +30,7 @@ matching_sources: # - EXTERNAL_DATABASE fully_included_sources: - # - CARMA # deprecated as no more public +#- CARMA # deprecated as no more public - GEO - GPD - GBPT diff --git a/test/config.test_myopic.yaml b/test/config.test_myopic.yaml index 758cbf1c..0f4c47f0 100644 --- a/test/config.test_myopic.yaml +++ b/test/config.test_myopic.yaml @@ -8,8 +8,7 @@ costs_dir: data/ #TODO change to the equivalent of technology data run: name: "test_myopic" # use this to keep track of runs with different settings name_subworkflow: "tutorial" # scenario name of the pypsa-earth subworkflow - shared_cutouts: true # set to true to share the default cutout(s) across runs - # Note: value false requires build_cutout to be enabled + shared_cutouts: true # set to true to share the default cutout(s) across runs; Note: value false requires build_cutout to be enabled foresight: myopic # option to disable the subworkflow to ease the analyses @@ -163,26 +162,26 @@ sector: district_heating: potential: 0.3 #maximum fraction of urban demand which can be supplied by district heating - #increase of today's district heating demand to potential maximum district heating share - #progress = 0 means today's district heating share, progress=-1 means maxumzm fraction of urban demand is supplied by district heating + #increase of today's district heating demand to potential maximum district heating share + #progress = 0 means today's district heating share, progress=-1 means maxumzm fraction of urban demand is supplied by district heating progress: 1 - #2020: 0.0 - #2030: 0.3 - #2040: 0.6 - #2050: 1.0 + # 2020: 0.0 + # 2030: 0.3 + # 2040: 0.6 + # 2050: 1.0 district_heating_loss: 0.15 reduce_space_heat_exogenously: true # reduces space heat demand by a given factor (applied before losses in DH) # this can represent e.g. building renovation, building demolition, or if # the factor is negative: increasing floor area, increased thermal comfort, population growth reduce_space_heat_exogenously_factor: 0.29 # per unit reduction in space heat demand # the default factors are determined by the LTS scenario from http://tool.european-calculator.eu/app/buildings/building-types-area/?levers=1ddd4444421213bdbbbddd44444ffffff11f411111221111211l212221 - # 2020: 0.10 # this results in a space heat demand reduction of 10% - # 2025: 0.09 # first heat demand increases compared to 2020 because of larger floor area per capita - # 2030: 0.09 - # 2035: 0.11 - # 2040: 0.16 - # 2045: 0.21 - # 2050: 0.29 + # 2020: 0.10 # this results in a space heat demand reduction of 10% + # 2025: 0.09 # first heat demand increases compared to 2020 because of larger floor area per capita + # 2030: 0.09 + # 2035: 0.11 + # 2040: 0.16 + # 2045: 0.21 + # 2050: 0.29 tes: true tes_tau: # 180 day time constant for centralised, 3 day for decentralised From 614e6908a133596c608e8f82f08d35863f0adead Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 9 Aug 2024 12:03:43 +0200 Subject: [PATCH 09/16] Snakefile: fix SDIR and RESDIR path ending --- Snakefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Snakefile b/Snakefile index 6d7fe8bc..0ff45a43 100644 --- a/Snakefile +++ b/Snakefile @@ -34,8 +34,10 @@ config["ROOT_PATH"] = os.getcwd() run = config.get("run", {}) RDIR = run["name"] + "/" if run.get("name") else "" CDIR = RDIR if not run.get("shared_cutouts") else "" -SDIR = str(Path(config["summary_dir"]) / RDIR) # use Path to flexibly allow closing "/" -RESDIR = str(Path(config["results_dir"]) / RDIR) +SDIR = ( + str(Path(config["summary_dir"]) / RDIR) + "/" +) # use Path to flexibly allow closing "/" +RESDIR = str(Path(config["results_dir"]) / RDIR) + "/" COSTDIR = config["costs_dir"] From 58a620d72348acda782c681240adf97645999072 Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 9 Aug 2024 12:12:51 +0200 Subject: [PATCH 10/16] ci.yaml include git submodule --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 57350bb6..2acdd89c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,6 +33,9 @@ jobs: steps: - uses: actions/checkout@v2 + - name: Fill submodule + run: | + git submodule update --init - name: Setup micromamba uses: mamba-org/setup-micromamba@v1 From 9de87d6bafa3609b1521b3e07fc9a9d39f8beaf4 Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 9 Aug 2024 12:21:25 +0200 Subject: [PATCH 11/16] fix duplicated "/" --- Snakefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Snakefile b/Snakefile index 0ff45a43..791b7ac7 100644 --- a/Snakefile +++ b/Snakefile @@ -556,7 +556,7 @@ if config["foresight"] == "overnight": network=RESDIR + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", costs=COSTDIR + "costs_{planning_horizons}.csv", - configs=SDIR + "/configs/config.yaml", # included to trigger copy_config rule + configs=SDIR + "configs/config.yaml", # included to trigger copy_config rule output: RESDIR + "postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", @@ -603,7 +603,7 @@ rule make_sector_summary: costs=COSTDIR + "costs_{planning_horizons}.csv", plots=expand( RESDIR - + "/maps/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}-costs-all_{planning_horizons}_{discountrate}_{demand}_{h2export}export.pdf", + + "maps/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}-costs-all_{planning_horizons}_{discountrate}_{demand}_{h2export}export.pdf", **config["scenario"], **config["costs"], **config["export"], @@ -640,7 +640,7 @@ rule plot_sector_network: + "postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", output: map=RESDIR - + "/maps/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}-costs-all_{planning_horizons}_{discountrate}_{demand}_{h2export}export.pdf", + + "maps/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}-costs-all_{planning_horizons}_{discountrate}_{demand}_{h2export}export.pdf", threads: 2 resources: mem_mb=10000, From 1219130640bf527ec3b61aff6cf32a4f2f8b39ad Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 9 Aug 2024 15:49:26 +0200 Subject: [PATCH 12/16] remove config.pypsa-earth.yaml in favour of actual pypsa-earth default config --- Snakefile | 2 +- config.pypsa-earth.yaml | 482 --------------------------------- scripts/prepare_gas_network.py | 1 - 3 files changed, 1 insertion(+), 484 deletions(-) delete mode 100644 config.pypsa-earth.yaml diff --git a/Snakefile b/Snakefile index 791b7ac7..8e31790c 100644 --- a/Snakefile +++ b/Snakefile @@ -18,7 +18,7 @@ if not exists("config.yaml"): copyfile("config.default.yaml", "config.yaml") -configfile: "config.pypsa-earth.yaml" +configfile: PYPSAEARTH_FOLDER + "/config.default.yaml" configfile: PYPSAEARTH_FOLDER + "/configs/bundle_config.yaml" configfile: PYPSAEARTH_FOLDER + "/configs/powerplantmatching_config.yaml" configfile: "config.yaml" diff --git a/config.pypsa-earth.yaml b/config.pypsa-earth.yaml deleted file mode 100644 index 235d384e..00000000 --- a/config.pypsa-earth.yaml +++ /dev/null @@ -1,482 +0,0 @@ -# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors -# -# SPDX-License-Identifier: CC0-1.0 - -version: 0.3.0 -tutorial: false - -logging: - level: INFO - format: "%(levelname)s:%(name)s:%(message)s" - -countries: ["MA"] -# Can be replaced by country ["NG", "BJ"], continent ["Africa"] or user-specific region, see more at https://pypsa-earth.readthedocs.io/en/latest/configuration.html#top-level-configuration - -enable: - retrieve_databundle: true # Recommended 'true', for the first run. Otherwise data might be missing. - retrieve_cost_data: true # true: retrieves cost data from technology data and saves in resources/costs.csv, false: uses cost data in data/costs.csv - download_osm_data: true # If 'true', OpenStreetMap data will be downloaded for the above given countries - build_natura_raster: false # If True, then an exclusion raster will be build - build_cutout: false - # If "build_cutout" : true, then environmental data is extracted according to `snapshots` date range and `countries` - # requires cds API key https://cds.climate.copernicus.eu/api-how-to - # More information https://atlite.readthedocs.io/en/latest/introduction.html#datasets - -custom_rules: [] # Default empty [] or link to custom rule file e.g. ["my_folder/my_rules.smk"] that add rules to Snakefile - -run: - name: "" # use this to keep track of runs with different settings - shared_cutouts: true # set to true to share the default cutout(s) across runs; Note: value false requires build_cutout to be enabled - -scenario: - simpl: [''] - ll: ['1.0'] - clusters: [10] - opts: [Co2L1-144H] - -snapshots: - start: "2013-01-01" - end: "2014-01-01" - inclusive: "left" # end is not inclusive - -# definition of the Coordinate Reference Systems -crs: - geo_crs: EPSG:4326 # general geographic projection, not used for metric measures. "EPSG:4326" is the standard used by OSM and google maps - distance_crs: EPSG:3857 # projection for distance measurements only. Possible recommended values are "EPSG:3857" (used by OSM and Google Maps) - area_crs: ESRI:54009 # projection for area measurements only. Possible recommended values are Global Mollweide "ESRI:54009" - -# download_osm_data_nprocesses: 10 # (optional) number of threads used to download osm data - -augmented_line_connection: - add_to_snakefile: false # If True, includes this rule to the workflow - connectivity_upgrade: 2 # Min. lines connection per node, https://networkx.org/documentation/stable/reference/algorithms/generated/networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation.html#networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation - new_line_type: ["HVAC"] # Expanded lines can be either ["HVAC"] or ["HVDC"] or both ["HVAC", "HVDC"] - min_expansion: 1 # [MW] New created line expands by float/int input - min_DC_length: 600 # [km] Minimum line length of DC line - -cluster_options: - simplify_network: - to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) - algorithm: kmeans # choose from: [hac, kmeans] - feature: solar+onwind-time # only for hac. choose from: [solar+onwind-time, solar+onwind-cap, solar-time, solar-cap, solar+offwind-cap] etc. - exclude_carriers: [] - remove_stubs: true - remove_stubs_across_borders: false - p_threshold_drop_isolated: 20 # [MW] isolated buses are being discarded if bus mean power is below the specified threshold - p_threshold_merge_isolated: 300 # [MW] isolated buses are being merged into a single isolated bus if a bus mean power is below the specified threshold - s_threshold_fetch_isolated: 0.05 # [-] a share of the national load for merging an isolated network into a backbone network - cluster_network: - algorithm: kmeans - feature: solar+onwind-time - exclude_carriers: [] - alternative_clustering: true # "False" use Voronoi shapes, "True" use GADM shapes - distribute_cluster: ['load'] # Distributes cluster nodes per country according to ['load'],['pop'] or ['gdp'] - out_logging: true # When "True", logging is printed to console - aggregation_strategies: - generators: # use "min" for more conservative assumptions - p_nom: sum - p_nom_max: sum - p_nom_min: sum - p_min_pu: mean - marginal_cost: mean - committable: any - ramp_limit_up: max - ramp_limit_down: max - efficiency: mean - -build_shape_options: - gadm_layer_id: 1 # GADM level area used for the gadm_shapes. Codes are country-dependent but roughly: 0: country, 1: region/county-like, 2: municipality-like - update_file: false # When true, all the input files are downloaded again and replace the existing files - out_logging: true # When true, logging is printed to console - year: 2020 # reference year used to derive shapes, info on population and info on GDP - nprocesses: 3 # number of processes to be used in build_shapes - worldpop_method: "standard" # "standard" pulls from web 1kmx1km raster, "api" pulls from API 100mx100m raster, false (not "false") no pop addition to shape which is useful when generating only cutout - gdp_method: "standard" # "standard" pulls from web 1x1km raster, false (not "false") no gdp addition to shape which useful when generating only cutout - contended_flag: "set_by_country" # "set_by_country" assigns the contended areas to the countries according to the GADM database, "drop" drops these contended areas from the model - -clean_osm_data_options: # osm = OpenStreetMap - names_by_shapes: true # Set the country name based on the extended country shapes - threshold_voltage: 51000 # [V] assets below that voltage threshold will not be used (cable, line, generator, etc.) - tag_substation: "transmission" # Filters only substations with 'transmission' tag, ('distribution' also available) - add_line_endings: true # When "True", then line endings are added to the dataset of the substations - generator_name_method: OSM # Methodology to specify the name to the generator. Options: OSM (name as by OSM dataset), closest_city (name by the closest city) - use_custom_lines: "OSM_only" # Use OSM (OSM_only), customized (custom_only), or both data sets (add_custom) - path_custom_lines: false # If exists, provide the specific absolute path of the custom file e.g. (...\data\custom_lines.geojson) - use_custom_substations: "OSM_only" # Use OSM (OSM_only), customized (custom_only), or both data sets (add_custom) - path_custom_substations: false # If exists, provide the specific absolute path of the custom file e.g. (...\data\custom_substations.geojson) - use_custom_cables: "OSM_only" # Use OSM (OSM_only), customized (custom_only), or both data sets (add_custom) - path_custom_cables: false # If exists, provide the specific absolute path of the custom file e.g. (...\data\custom_cables.geojson) - -build_osm_network: # Options of the build_osm_network script; osm = OpenStreetMap - group_close_buses: true # When "True", close buses are merged and guarantee the voltage matching among line endings - group_tolerance_buses: 10000 # [m] (default 5000) Tolerance in meters of the close buses to merge - split_overpassing_lines: true # When True, lines overpassing buses are splitted and connected to the bueses - overpassing_lines_tolerance: 1 # [m] (default 1) Tolerance to identify lines overpassing buses - force_ac: false # When true, it forces all components (lines and substation) to be AC-only. To be used if DC assets create problem. - -base_network: - min_voltage_substation_offshore: 51000 # [V] minimum voltage of the offshore substations - min_voltage_rebase_voltage: 51000 # [V] minimum voltage in base network - -load_options: - ssp: "ssp2-2.6" # shared socio-economic pathway (GDP and population growth) scenario to consider - weather_year: 2013 # Load scenarios available with different weather year (different renewable potentials) - prediction_year: 2030 # Load scenarios available with different prediction year (GDP, population) - scale: 1 # scales all load time-series, i.e. 2 = doubles load - -electricity: - base_voltage: 380. - voltages: [132., 220., 300., 380., 500., 750.] - co2limit: 7.75e+7 # European default, 0.05 * 3.1e9*0.5, needs to be adjusted for Africa - co2base: 1.487e+9 # European default, adjustment to Africa necessary - agg_p_nom_limits: data/agg_p_nom_minmax.csv - hvdc_as_lines: false # should HVDC lines be modeled as `Line` or as `Link` component? - automatic_emission: false - automatic_emission_base_year: 1990 # 1990 is taken as default. Any year from 1970 to 2018 can be selected. - - operational_reserve: # like https://genxproject.github.io/GenX/dev/core/#Reserves - activate: false - epsilon_load: 0.02 # share of total load - epsilon_vres: 0.02 # share of total renewable supply - contingency: 0 # fixed capacity in MW - - max_hours: - battery: 6 - H2: 168 - - extendable_carriers: - Generator: [solar, onwind, offwind-ac, offwind-dc, OCGT] - StorageUnit: [] # battery, H2 - Store: [] - Link: [] # H2 pipeline - - powerplants_filter: (DateOut >= 2022 or DateOut != DateOut) - custom_powerplants: false # "false" use only powerplantmatching (ppm) data, "merge" combines ppm and custom powerplants, "replace" use only custom powerplants - - conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass] - renewable_carriers: [solar, onwind, offwind-ac, offwind-dc, hydro] - - estimate_renewable_capacities: - stats: "irena" # False, = greenfield expansion, 'irena' uses IRENA stats to add expansion limits - year: 2020 # Reference year, available years for IRENA stats are 2000 to 2020 - p_nom_min: 1 # any float, scales the minimum expansion acquired from stats, i.e. 110% of 's capacities => p_nom_min: 1.1 - p_nom_max: false # sets the expansion constraint, False to deactivate this option and use estimated renewable potentials determine by the workflow, float scales the p_nom_min factor accordingly - technology_mapping: - # Wind is the Fueltype in ppm.data.Capacity_stats, onwind, offwind-{ac,dc} the carrier in PyPSA-Earth - Offshore: [offwind-ac, offwind-dc] - Onshore: [onwind] - PV: [solar] - -lines: - ac_types: - 132.: "243-AL1/39-ST1A 20.0" - 220.: "Al/St 240/40 2-bundle 220.0" - 300.: "Al/St 240/40 3-bundle 300.0" - 380.: "Al/St 240/40 4-bundle 380.0" - 500.: "Al/St 240/40 4-bundle 380.0" - 750.: "Al/St 560/50 4-bundle 750.0" - dc_types: - 500.: "HVDC XLPE 1000" - s_max_pu: 0.7 - s_nom_max: .inf - length_factor: 1.25 - under_construction: "zero" # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity - -links: - p_max_pu: 1.0 - p_nom_max: .inf - under_construction: "zero" # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity - -transformers: - x: 0.1 - s_nom: 2000. - type: "" - -atlite: - nprocesses: 4 - cutouts: - # geographical bounds automatically determined from countries input - cutout-2013-era5: - module: era5 - dx: 0.3 # cutout resolution - dy: 0.3 # cutout resolution - # The cutout time is automatically set by the snapshot range. See `snapshot:` option above and 'build_cutout.py'. - # time: ["2013-01-01", "2014-01-01"] # to manually specify a different weather year (~70 years available) - # The cutout spatial extent [x,y] is automatically set by country selection. See `countries:` option above and 'build_cutout.py'. - # x: [-12., 35.] # set cutout range manual, instead of automatic by boundaries of country - # y: [33., 72] # manual set cutout range - - -renewable: - onwind: - cutout: cutout-2013-era5 - resource: - method: wind - turbine: Vestas_V112_3MW - capacity_per_sqkm: 3 # conservative, ScholzPhd Tab 4.3.1: 10MW/km^2 - # correction_factor: 0.93 - copernicus: - # Scholz, Y. (2012). Renewable energy based electricity supply at low costs: - # development of the REMix model and application for Europe. ( p.42 / p.28) - # CLC grid codes: - # 11X/12X - Various forest types - # 20 - Shrubs - # 30 - Herbaceus vegetation - # 40 - Cropland - # 50 - Urban - # 60 - Bare / Sparse vegetation - # 80 - Permanent water bodies - # 100 - Moss and lichen - # 200 - Open sea - grid_codes: [20, 30, 40, 60, 100, 111, 112, 113, 114, 115, 116, 121, 122, 123, 124, 125, 126] - distance: 1000 - distance_grid_codes: [50] - natura: true - potential: simple # or conservative - clip_p_max_pu: 1.e-2 - extendable: true - offwind-ac: - cutout: cutout-2013-era5 - resource: - method: wind - turbine: NREL_ReferenceTurbine_5MW_offshore - capacity_per_sqkm: 2 - correction_factor: 0.8855 - # proxy for wake losses - # from 10.1016/j.energy.2018.08.153 - # until done more rigorously in #153 - copernicus: - grid_codes: [80, 200] - natura: true - max_depth: 50 - max_shore_distance: 30000 - potential: simple # or conservative - clip_p_max_pu: 1.e-2 - extendable: true - offwind-dc: - cutout: cutout-2013-era5 - resource: - method: wind - turbine: NREL_ReferenceTurbine_5MW_offshore - # ScholzPhd Tab 4.3.1: 10MW/km^2 - capacity_per_sqkm: 3 - correction_factor: 0.8855 - # proxy for wake losses - # from 10.1016/j.energy.2018.08.153 - # until done more rigorously in #153 - copernicus: - grid_codes: [80, 200] - natura: true - max_depth: 50 - min_shore_distance: 30000 - potential: simple # or conservative - clip_p_max_pu: 1.e-2 - extendable: true - solar: - cutout: cutout-2013-era5 - resource: - method: pv - panel: CSi - orientation: latitude_optimal # will lead into optimal design - # slope: 0. # slope: 0 represent a flat panel - # azimuth: 180. # azimuth: 180 south orientation - capacity_per_sqkm: 4.6 # From 1.7 to 4.6 addresses issue #361 - # Determined by comparing uncorrected area-weighted full-load hours to those - # published in Supplementary Data to - # Pietzcker, Robert Carl, et al. "Using the sun to decarbonize the power - # sector: The economic potential of photovoltaics and concentrating solar - # power." Applied Energy 135 (2014): 704-720. - correction_factor: 0.854337 - copernicus: - grid_codes: [20, 30, 40, 50, 60, 90, 100] - natura: true - potential: simple # or conservative - clip_p_max_pu: 1.e-2 - extendable: true - hydro: - cutout: cutout-2013-era5 - hydrobasins_level: 6 - resource: - method: hydro - hydrobasins: data/hydrobasins/hybas_world.shp - flowspeed: 1.0 # m/s - # weight_with_height: false - # show_progress: true - carriers: [ror, PHS, hydro] - PHS_max_hours: 6 - hydro_max_hours: "energy_capacity_totals_by_country" # not active - hydro_max_hours_default: 6.0 # (optional, default 6) Default value of max_hours for hydro when NaN values are found - clip_min_inflow: 1.0 - extendable: true - normalization: - method: hydro_capacities # 'hydro_capacities' to rescale country hydro production by using hydro_capacities, 'eia' to rescale by eia data, false for no rescaling - year: 2013 # (optional) year of statistics used to rescale the runoff time series. When not provided, the weather year of the snapshots is used - multiplier: 1.1 # multiplier applied after the normalization of the hydro production; default 1.0 - -# TODO: Needs to be adjusted for Africa. -# Costs Configuration (Do not remove, needed for Sphynx documentation). -costs: - year: 2030 - version: v0.5.0 - rooftop_share: 0.14 # based on the potentials, assuming (0.1 kW/m2 and 10 m2/person) - USD2013_to_EUR2013: 0.7532 # [EUR/USD] ECB: https://www.ecb.europa.eu/stats/exchange/eurofxref/html/eurofxref-graph-usd.en.html - fill_values: - FOM: 0 - VOM: 0 - efficiency: 1 - fuel: 0 - investment: 0 - lifetime: 25 - CO2 intensity: 0 - discount rate: 0.071 - marginal_cost: # EUR/MWh - solar: 0.01 - onwind: 0.015 - offwind: 0.015 - hydro: 0. - H2: 0. - electrolysis: 0. - fuel cell: 0. - battery: 0. - battery inverter: 0. - emission_prices: # in currency per tonne emission, only used with the option Ep - co2: 0. - - -monte_carlo: - # Description: Specify Monte Carlo sampling options for uncertainty analysis. - # Define the option list for Monte Carlo sampling. - # Make sure add_to_snakefile is set to true to enable Monte-Carlo - options: - add_to_snakefile: false # When set to true, enables Monte Carlo sampling - samples: 9 # number of optimizations. Note that number of samples when using scipy has to be the square of a prime number - sampling_strategy: "chaospy" # "pydoe2", "chaospy", "scipy", packages that are supported - seed: 42 # set seedling for reproducibility - # Uncertainties on any PyPSA object are specified by declaring the specific PyPSA object under the key 'uncertainties'. - # For each PyPSA object, the 'type' and 'args' keys represent the type of distribution and its argument, respectively. - # Supported distributions types are uniform, normal, lognormal, triangle, beta and gamma. - # The arguments of the distribution are passed using the key 'args' as follows, tailored by distribution type - # normal: [mean, std], lognormal: [mean, std], uniform: [lower_bound, upper_bound], - # triangle: [mid_point (between 0 - 1)], beta: [alpha, beta], gamma: [shape, scale] - # More info on the distributions are documented in the Chaospy reference guide... - # https://chaospy.readthedocs.io/en/master/reference/distribution/index.html - # An abstract example is as follows: - # {pypsa network object, e.g. "loads_t.p_set"}: - # type: {any supported distribution among the previous: "uniform", "normal", ...} - # args: {arguments passed as a list depending on the distribution, see the above and more at https://pypsa.readthedocs.io/} - uncertainties: - loads_t.p_set: - type: uniform - args: [0, 1] - generators_t.p_max_pu.loc[:, n.generators.carrier == "onwind"]: - type: lognormal - args: [1.5] - generators_t.p_max_pu.loc[:, n.generators.carrier == "solar"]: - type: beta - args: [0.5, 2] - - - -solving: - options: - formulation: kirchhoff - load_shedding: true - noisy_costs: true - min_iterations: 4 - max_iterations: 6 - clip_p_max_pu: 0.01 - skip_iterations: true - track_iterations: false - #nhours: 10 - solver: - name: gurobi - threads: 4 - method: 2 # barrier (=ipm) - crossover: 0 - BarConvTol: 1.e-5 - FeasibilityTol: 1.e-6 - AggFill: 0 - PreDual: 0 - GURO_PAR_BARDENSETHRESH: 200 - - -plotting: - map: - figsize: [7, 7] - boundaries: [-10.2, 29, 35, 72] - p_nom: - bus_size_factor: 5.e+4 - linewidth_factor: 3.e+3 - - costs_max: 800 - costs_threshold: 1 - - energy_max: 15000. - energy_min: -10000. - energy_threshold: 50. - - vre_techs: ["onwind", "offwind-ac", "offwind-dc", "solar", "ror"] - conv_techs: ["OCGT", "CCGT", "nuclear", "coal", "oil"] - storage_techs: ["hydro+PHS", "battery", "H2"] - load_carriers: ["AC load"] - AC_carriers: ["AC line", "AC transformer"] - link_carriers: ["DC line", "Converter AC-DC"] - tech_colors: - "onwind": "#235ebc" - "onshore wind": "#235ebc" - "offwind": "#6895dd" - "offwind-ac": "#6895dd" - "offshore wind": "#6895dd" - "offshore wind ac": "#6895dd" - "offwind-dc": "#74c6f2" - "offshore wind dc": "#74c6f2" - "hydro": "#08ad97" - "hydro+PHS": "#08ad97" - "PHS": "#08ad97" - "hydro reservoir": "#08ad97" - "hydroelectricity": "#08ad97" - "ror": "#4adbc8" - "run of river": "#4adbc8" - "solar": "#f9d002" - "solar PV": "#f9d002" - "solar thermal": "#ffef60" - "biomass": "#0c6013" - "solid biomass": "#06540d" - "biogas": "#23932d" - "waste": "#68896b" - "geothermal": "#ba91b1" - "OCGT": "#d35050" - "gas": "#d35050" - "natural gas": "#d35050" - "CCGT": "#b20101" - "nuclear": "#ff9000" - "coal": "#707070" - "lignite": "#9e5a01" - "oil": "#262626" - "H2": "#ea048a" - "hydrogen storage": "#ea048a" - "battery": "#b8ea04" - "Electric load": "#f9d002" - "electricity": "#f9d002" - "lines": "#70af1d" - "transmission lines": "#70af1d" - "AC-AC": "#70af1d" - "AC line": "#70af1d" - "links": "#8a1caf" - "HVDC links": "#8a1caf" - "DC-DC": "#8a1caf" - "DC link": "#8a1caf" - "load": "#FF0000" - nice_names: - OCGT: "Open-Cycle Gas" - CCGT: "Combined-Cycle Gas" - offwind-ac: "Offshore Wind (AC)" - offwind-dc: "Offshore Wind (DC)" - onwind: "Onshore Wind" - solar: "Solar" - PHS: "Pumped Hydro Storage" - hydro: "Reservoir & Dam" - battery: "Battery Storage" - H2: "Hydrogen Storage" - lines: "Transmission Lines" - ror: "Run of River" diff --git a/scripts/prepare_gas_network.py b/scripts/prepare_gas_network.py index f8833b3a..1849429f 100644 --- a/scripts/prepare_gas_network.py +++ b/scripts/prepare_gas_network.py @@ -346,7 +346,6 @@ def download_GADM(country_code, update=False, out_logging=False): GADM_filename = get_GADM_filename(country_code) GADM_inputfile_gpkg = os.path.join( - os.getcwd() + "/pypsa-earth", "data", "gadm", GADM_filename, From 6c241aec84fb4124eec1a2d496890c51cb5912e1 Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 12 Aug 2024 17:52:04 +0200 Subject: [PATCH 13/16] remove pypsa-earth.config from copy_config --- scripts/copy_config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/copy_config.py b/scripts/copy_config.py index 4848c90a..edc6bddd 100644 --- a/scripts/copy_config.py +++ b/scripts/copy_config.py @@ -7,7 +7,6 @@ "./Snakefile": "Snakefile", "./scripts/solve_network.py": "solve_network.py", "./scripts/prepare_sector_network.py": "prepare_sector_network.py", - "./config.pypsa-earth.yaml": "config.pypsa-earth.yaml", } if __name__ == "__main__": From f229301e33c13bf1a9211884871f7df60e03ab7d Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 12 Aug 2024 19:05:34 +0200 Subject: [PATCH 14/16] ci: use one core to better track log --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3729944f..01ac167f 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ test: set -e - snakemake -call solve_sector_networks --configfile pypsa-earth/config.tutorial.yaml test/config.test1.yaml + snakemake -c1 solve_sector_networks --configfile pypsa-earth/config.tutorial.yaml test/config.test1.yaml echo "All tests completed successfully." setup: From 94ba37036bc14191847a737779d88eac29ec2195 Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 13 Aug 2024 11:59:39 +0200 Subject: [PATCH 15/16] test: use more core --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 01ac167f..b5ad66ed 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ test: set -e - snakemake -c1 solve_sector_networks --configfile pypsa-earth/config.tutorial.yaml test/config.test1.yaml + snakemake -c4 solve_sector_networks --configfile pypsa-earth/config.tutorial.yaml test/config.test1.yaml echo "All tests completed successfully." setup: From ef54888b203612a70e1c0111f9894250e14d6679 Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 13 Aug 2024 17:24:30 +0200 Subject: [PATCH 16/16] update submodule --- pypsa-earth | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypsa-earth b/pypsa-earth index 23bc59fe..d8c6351c 160000 --- a/pypsa-earth +++ b/pypsa-earth @@ -1 +1 @@ -Subproject commit 23bc59feceed82eee9270111140fba55ec0f915a +Subproject commit d8c6351c5ef9133a0ee35fd5159408c494b9bd09