Merge branch 'master' into feature/overwrite-cost-attributes

.github/workflows/test.yaml

@@ -85,6 +85,11 @@ jobs:
         conda env update -n pypsa-eur -f ${{ env.env_file }}
         echo "Run conda list" && conda list
 
+    - name: Run pylint check on scripts
+      # check for undefined variables to reuse functions across scripts
+      run: |
+        pylint --disable=all --enable=E0601 --output-format=parseable scripts/add_* scripts/prepare_* scripts/solve_* 
+
     - name: Run snakemake test workflows
       run: |
         make test

.github/workflows/update-pinned-env.yaml

@@ -83,5 +83,8 @@ jobs:
         token: ${{ secrets.GITHUB_TOKEN }}
         branch: update-pinned-environment
         title: "[github-actions.ci] Update pinned envs"
-        body: "Automatically generated PR to update pinned environment files for Windows, macOS, and Linux."
+        body: |
+          Automatically generated PR to update pinned environment files for Windows, macOS, and Linux.
+
+          **Note: Do not merge without manual test execution. Either update the branch to trigger tests, or use `workflow_dispatch` to run tests manually. Unlike standard PRs, tests will not run automatically.**
         commit-message: "Update pinned environment files for all platforms"

.github/workflows/validate.yaml

@@ -14,19 +14,33 @@ jobs:
     name: Run validation
     if: github.event.pull_request.head.repo.full_name == github.repository
     runs-on: self-hosted
+    outputs:
+      validation_failed: ${{ steps.check_validation.outputs.failed }}
     steps:
     - uses: lkstrp/[email protected]
+      id: validation
+      continue-on-error: true
       with:
         step: run-self-hosted-validation
         env_file: envs/environment.yaml
         snakemake_config: config/test/config.validator.yaml
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    - name: Check validation status
+      id: check_validation
+      run: |
+        if [[ "${{ steps.validation.outcome }}" == "failure" ]]; then
+          echo "failed=true" >> $GITHUB_OUTPUT
+        else
+          echo "failed=false" >> $GITHUB_OUTPUT
+        fi
 
   create-report:
     name: Create report
-    if: github.event.pull_request.head.repo.full_name == github.repository
     needs: run-validation
+    if: |
+      github.event.pull_request.head.repo.full_name == github.repository && 
+      needs.run-validation.outputs.validation_failed != 'true'
     runs-on: ubuntu-latest
     steps:
     - uses: lkstrp/[email protected]

.pre-commit-config.yaml

@@ -14,7 +14,7 @@ repos:
 # Run ruff to lint and format
 - repo: https://github.com/astral-sh/ruff-pre-commit
   # Ruff version.
-  rev: v0.9.3
+  rev: v0.9.6
   hooks:
     # Run the linter.
   - id: ruff
@@ -24,7 +24,7 @@ repos:
 
   # Find common spelling mistakes in comments and docstrings
 - repo: https://github.com/codespell-project/codespell
-  rev: v2.4.0
+  rev: v2.4.1
   hooks:
   - id: codespell
     args: ['--ignore-regex="(\b[A-Z]+\b)"', '--ignore-words-list=fom,appartment,bage,ore,setis,tabacco,berfore,vor,pris'] # Ignore capital case words, e.g. country codes

Snakefile

@@ -10,7 +10,13 @@ from snakemake.utils import min_version
 
 min_version("8.11")
 
-from scripts._helpers import path_provider, copy_default_files, get_scenarios, get_rdir
+from scripts._helpers import (
+    path_provider,
+    copy_default_files,
+    get_scenarios,
+    get_rdir,
+    get_shadow,
+)
 
 
 copy_default_files(workflow)
@@ -23,6 +29,7 @@ configfile: "config/config.yaml"
 run = config["run"]
 scenarios = get_scenarios(run)
 RDIR = get_rdir(run)
+shadow_config = get_shadow(run)
 
 shared_resources = run["shared_resources"]["policy"]
 exclude_from_shared = run["shared_resources"]["exclude"]

config/config.default.yaml

@@ -30,6 +30,7 @@ run:
     policy: false
     exclude: []
   shared_cutouts: true
+  use_shadow_directory: true # Set to false if problems regarding missing directories occur
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#foresight
 foresight: overnight
@@ -481,16 +482,9 @@ sector:
       heat_exchanger_pinch_point_temperature_difference: 5 #K
       isentropic_compressor_efficiency: 0.8
       heat_loss: 0.0
-    heat_utilisation_potentials:
+    limited_heat_sources:
       geothermal:
-        # activate for 85C hydrothermal
-        # key: hydrothermal_85
-        # constant_temperature_celsius: 85
-        key: hydrothermal_65
         constant_temperature_celsius: 65
-        column_name: Energy_TWh
-        unit: TWh
-        full_load_hours: 4000
     direct_utilisation_heat_sources:
     - geothermal
   heat_pump_sources:

doc/configtables/run.csv

@@ -9,3 +9,4 @@ shared_resources,,,
 -- policy,bool/str,,"Boolean switch to select whether resources should be shared across runs. If a string is passed, this is used as a subdirectory name for shared resources. If set to 'base', only resources before creating the elec.nc file are shared."
 -- exclude,str,"For the case shared_resources=base, specify additional files that should not be shared across runs."
 shared_cutouts,bool,"{true, false}","Switch to select whether cutouts should be shared across runs."
+use_shadow_directory,bool,"{true, false}","Set to ``true`` (default) if snakemake shadow directories (``shallow``) should be used. Set to ``false`` if problems occcur."

doc/configtables/sector.csv

@@ -23,13 +23,9 @@ district_heating,--,,`prepare_sector_network.py <https://github.com/PyPSA/pypsa-
 -- -- heat_exchanger_pinch_point_temperature_difference,K,float,Heat pump pinch point temperature difference in heat exchangers assumed for approximation.
 -- -- isentropic_compressor_efficiency,--,float,Isentropic efficiency of heat pump compressor assumed for approximation. Must be between 0 and 1.
 -- -- heat_loss,--,float,Heat pump heat loss assumed for approximation. Must be between 0 and 1.
--- heat_utilisation_potentials,--,Dictionary with names of heat sources for which data by Fraunhofer ISI (`Manz et al. 2024 <https://www.sciencedirect.com/science/article/pii/S0960148124001769>) should be used,
+-- limited_heat_sources,--,Dictionary with names of limited heat sources (not air) for which data by Fraunhofer ISI (`Manz et al. 2024 <https://www.sciencedirect.com/science/article/pii/S0960148124001769>) is used,
 -- -- geothermal,-,Name of the heat source. Must be the same as in ``heat_pump_sources``,
--- -- -- key,-,string used to complete URL for data download - e.g. `geothermal_65` or `geothermal_85`","i.e file names in `Fordatis <https://fordatis.fraunhofer.de/handle/fordatis/341.3?mode=simple>`"
--- -- -- constant_temperature_celsius,C,heat source temperature,
--- -- -- column_name,-,name of the data column in retrieved GeoDataFrame,
--- -- -- unit,-,unit of heat source potential must be in (K/M/G/T)Wh,
--- -- -- full_load_hours,h,assumed full-load hours in Manz et al. (used to scale from utilisation to technical potential),
+-- -- -- constant_temperature_celsius,°C,heat source temperature,
 -- direct_utilisation_heat_sources,--,List of heat sources for direct heat utilisation in district heating. Must be in the keys of `heat_utilisation_potentials` (e.g. ``geothermal``),
 -- heat_pump_sources,--,,
 -- -- urban central,--,List of heat sources for heat pumps in urban central heating,

doc/release_notes.rst

@@ -13,7 +13,11 @@ Upcoming Release
 
 - Added option to specify the cutout directory in the configuration file. This allows to the user to specify the directory where the cutouts are stored. Use it by setting ``atlite: cutout_directory:`` in the configuration file. (https://github.com/PyPSA/pypsa-eur/pull/1515)
 * Add the options to overwrite investment, lifetime, FOM, VOM, efficiency and fuel attributes from the configuration file. This mimics the existing capital and marginal cost behaviour. (https://github.com/PyPSA/pypsa-eur/pull/1532)
+* Bugfix: Geothermal heat potentials are now restricted to those in close proximity to future district heating areas as projected by Manz et al. 2024. Includes a refactoring change: Building of generic technical potentials from heat utilisation potentials was changed to specific computation of geothermal heat potentials.
 
+- ...
+
+- Bug fix: Added setting ``run: use_shadow_directory:`` (default: ``true``) which sets the ``shadow`` parameter of the snakemake workflow. Configuring to ``true`` sets snakemake ``shadow`` parameter to ``shalloow``, ``false`` to `Ǹone``. Should be set to ``false`` for those cases, where snakemake has an issue with finding missing input/output files in solving rules.
 
 PyPSA-Eur v2025.01.0 (24th January 2025)
 ========================================

envs/environment.yaml

@@ -59,6 +59,7 @@ dependencies:
 - ipython
 - pre-commit
 - ruff
+- pylint
 
 - pip:
   - gurobipy