Divide up integration tests (#224)

* refactor tests, and only run integration tests on `main`

* set test dir in step 1

* add runs-on

* divide out workflows

* format

* tidy

* fix

* use tests dir

* add comment

* change check to repo owner

* tidy

* change to !- pull request

* update comment

* echo

* use github output

* run second job if push

* format

* pytest.mark.integration

* add timeout

* remove timeout from merge

* turn normalised error into percent

* limit metrics to 3sf

* update site used in test_evaluation script to give non nan result

* add pytest.ini file to mark integration tests

* update forecast test with no ts

* update forecast test with no ts spelling

---------

Co-authored-by: Megawattz <[email protected]>

Author: peterdudfield

.github/workflows/pytest.yaml

@@ -8,13 +8,28 @@ on:
     - cron: "0 12 * * 1"
   pull_request_target:
     types: [opened, synchronize, reopened, ready_for_review]
+
 jobs:
-  call-run-python-tests:
+  call-run-python-tests-unit:
+    uses: openclimatefix/.github/.github/workflows/python-test.yml@issue/pip-all
+    with:
+      # pytest-cov looks at this folder
+      pytest_cov_dir: "quartz_solar_forecast"
+      os_list: '["ubuntu-latest"]'
+      python-version: "['3.11']"
+      extra_commands: echo "HF_TOKEN=${{ vars.HF_TOKEN }}" > .env
+      pytest_numcpus: '1'
+      test_dir: tests/unit
+
+  call-run-python-tests-all:
+    # only run on push, not external PR
     uses: openclimatefix/.github/.github/workflows/python-test.yml@issue/pip-all
+    if: github.event_name == 'push'
     with:
       # pytest-cov looks at this folder
       pytest_cov_dir: "quartz_solar_forecast"
       os_list: '["ubuntu-latest"]'
       python-version: "['3.11']"
       extra_commands: echo "HF_TOKEN=${{ vars.HF_TOKEN }}" > .env
       pytest_numcpus: '1'
+      test_dir: tests

pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+markers =
+    integration: marks tests as integration tests

quartz_solar_forecast/eval/metrics.py

@@ -35,7 +35,7 @@ def metrics(results_df: pd.DataFrame, pv_metadata: pd.DataFrame, include_night:
         .mean(),
         4,
     )
-    print(f"MAE: {mae} kw, normalized {mae_normalized} %")
+    print(f"MAE: {mae} kw, normalized {100*mae_normalized} %")
 
     # calculate metrics over the different horizons hours
     # find all unique horizon_hours
@@ -74,7 +74,7 @@ def metrics(results_df: pd.DataFrame, pv_metadata: pd.DataFrame, include_night:
         )
 
         print(
-            f"MAE for horizon {horizon_group}: {mae} +- {1.96*sem}. mae_normalized: {100*mae_normalized} %"
+            f"MAE for horizon {horizon_group}: {mae} +- {1.96*sem:.3g}. mae_normalized: {100*mae_normalized:.3g} %"
         )
 
         # TODO add more metrics using ocf_ml_metrics

tests/eval/test_evaluation.py renamed to tests/integration/eval/test_evaluation.py

@@ -1,8 +1,10 @@
 from quartz_solar_forecast.evaluation import run_eval
 import tempfile
 import pandas as pd
+import pytest
 
 
+@pytest.mark.integration
 def test_run_eval():
 
     # create a fake dataframe
@@ -15,7 +17,7 @@ def test_run_eval():
                 "pv_id",
                 "timestamp",
             ],
-            data=[[8215, "2021-01-26 01:15:00"], [8215, "2021-01-30 16:30:00"]],
+            data=[[7593, "2021-08-21 12:00:00"], [7593, "2021-10-04 20:00:00"]],
         )
 
         testset_filename = tmpdirname + "/test_dataset.csv"

tests/eval/test_nwp.py renamed to tests/integration/eval/test_nwp.py

@@ -1,8 +1,10 @@
 from quartz_solar_forecast.eval.nwp import get_nwp
 import pandas as pd
+import pytest
 
 
 # can take ~ 1 minute to run
+@pytest.mark.integration
 def test_get_nwp():
     # make test dataset file
     test_set_df = pd.DataFrame(

tests/eval/test_pv.py renamed to tests/integration/eval/test_pv.py

@@ -1,7 +1,8 @@
 from quartz_solar_forecast.eval.pv import get_pv_truth, get_pv_metadata
 import pandas as pd
+import pytest
 
-
+@pytest.mark.integration
 def test_get_pv_metadata():
     test_set_df = pd.DataFrame(
         [
@@ -16,6 +17,7 @@ def test_get_pv_metadata():
     assert "latitude" in metadata_df.columns
 
 
+@pytest.mark.integration
 def test_get_pv():
     # make test dataset file
     test_set_df = pd.DataFrame(