From 04b43a8ef852dd943687924e0852dd9c674258ba Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Mon, 18 Nov 2024 20:20:52 +0000 Subject: [PATCH 01/26] refactor tests, and only run integration tests on `main` --- .github/workflows/pytest.yaml | 2 ++ tests/{ => integration}/eval/test_evaluation.py | 2 ++ tests/{ => integration}/eval/test_nwp.py | 2 ++ tests/{ => integration}/eval/test_pv.py | 4 +++- tests/{ => unit}/api_tests/test_api.py | 0 tests/{ => unit}/data/test_make_test_set.py | 0 tests/{ => unit}/data/test_process_pv_data.py | 0 tests/{ => unit}/eval/test_eval_forecast.py | 0 tests/{ => unit}/eval/test_metrics.py | 0 tests/{ => unit}/inverters/test_process_enphase_data.py | 0 tests/{ => unit}/inverters/test_victron.py | 0 tests/{ => unit}/test_forecast.py | 0 tests/{ => unit}/test_forecast_no_ts.py | 0 tests/{ => unit}/test_generate_forecast.py | 0 tests/{ => unit}/utils/test_file_path.py | 0 15 files changed, 9 insertions(+), 1 deletion(-) rename tests/{ => integration}/eval/test_evaluation.py (94%) rename tests/{ => integration}/eval/test_nwp.py (92%) rename tests/{ => integration}/eval/test_pv.py (91%) rename tests/{ => unit}/api_tests/test_api.py (100%) rename tests/{ => unit}/data/test_make_test_set.py (100%) rename tests/{ => unit}/data/test_process_pv_data.py (100%) rename tests/{ => unit}/eval/test_eval_forecast.py (100%) rename tests/{ => unit}/eval/test_metrics.py (100%) rename tests/{ => unit}/inverters/test_process_enphase_data.py (100%) rename tests/{ => unit}/inverters/test_victron.py (100%) rename tests/{ => unit}/test_forecast.py (100%) rename tests/{ => unit}/test_forecast_no_ts.py (100%) rename tests/{ => unit}/test_generate_forecast.py (100%) rename tests/{ => unit}/utils/test_file_path.py (100%) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index ec5addb6..40dfe4c1 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -18,3 +18,5 @@ jobs: python-version: "['3.11']" extra_commands: echo "HF_TOKEN=${{ vars.HF_TOKEN }}" > .env pytest_numcpus: '1' + # if branch is 'main' then run all tests, if not only run unittests + test_dir: ${{ github.ref == 'refs/heads/main' && '' || 'tests/unit' }} diff --git a/tests/eval/test_evaluation.py b/tests/integration/eval/test_evaluation.py similarity index 94% rename from tests/eval/test_evaluation.py rename to tests/integration/eval/test_evaluation.py index e6548b2b..ce94986d 100644 --- a/tests/eval/test_evaluation.py +++ b/tests/integration/eval/test_evaluation.py @@ -1,8 +1,10 @@ from quartz_solar_forecast.evaluation import run_eval import tempfile import pandas as pd +import pytest +@pytest.test.integration def test_run_eval(): # create a fake dataframe diff --git a/tests/eval/test_nwp.py b/tests/integration/eval/test_nwp.py similarity index 92% rename from tests/eval/test_nwp.py rename to tests/integration/eval/test_nwp.py index e9c3659b..46b786cc 100644 --- a/tests/eval/test_nwp.py +++ b/tests/integration/eval/test_nwp.py @@ -1,8 +1,10 @@ from quartz_solar_forecast.eval.nwp import get_nwp import pandas as pd +import pytest # can take ~ 1 minute to run +@pytest.test.integration def test_get_nwp(): # make test dataset file test_set_df = pd.DataFrame( diff --git a/tests/eval/test_pv.py b/tests/integration/eval/test_pv.py similarity index 91% rename from tests/eval/test_pv.py rename to tests/integration/eval/test_pv.py index 51739547..56698c6e 100644 --- a/tests/eval/test_pv.py +++ b/tests/integration/eval/test_pv.py @@ -1,7 +1,8 @@ from quartz_solar_forecast.eval.pv import get_pv_truth, get_pv_metadata import pandas as pd +import pytest - +@pytest.mark.integration def test_get_pv_metadata(): test_set_df = pd.DataFrame( [ @@ -16,6 +17,7 @@ def test_get_pv_metadata(): assert "latitude" in metadata_df.columns +@pytest.mark.integration def test_get_pv(): # make test dataset file test_set_df = pd.DataFrame( diff --git a/tests/api_tests/test_api.py b/tests/unit/api_tests/test_api.py similarity index 100% rename from tests/api_tests/test_api.py rename to tests/unit/api_tests/test_api.py diff --git a/tests/data/test_make_test_set.py b/tests/unit/data/test_make_test_set.py similarity index 100% rename from tests/data/test_make_test_set.py rename to tests/unit/data/test_make_test_set.py diff --git a/tests/data/test_process_pv_data.py b/tests/unit/data/test_process_pv_data.py similarity index 100% rename from tests/data/test_process_pv_data.py rename to tests/unit/data/test_process_pv_data.py diff --git a/tests/eval/test_eval_forecast.py b/tests/unit/eval/test_eval_forecast.py similarity index 100% rename from tests/eval/test_eval_forecast.py rename to tests/unit/eval/test_eval_forecast.py diff --git a/tests/eval/test_metrics.py b/tests/unit/eval/test_metrics.py similarity index 100% rename from tests/eval/test_metrics.py rename to tests/unit/eval/test_metrics.py diff --git a/tests/inverters/test_process_enphase_data.py b/tests/unit/inverters/test_process_enphase_data.py similarity index 100% rename from tests/inverters/test_process_enphase_data.py rename to tests/unit/inverters/test_process_enphase_data.py diff --git a/tests/inverters/test_victron.py b/tests/unit/inverters/test_victron.py similarity index 100% rename from tests/inverters/test_victron.py rename to tests/unit/inverters/test_victron.py diff --git a/tests/test_forecast.py b/tests/unit/test_forecast.py similarity index 100% rename from tests/test_forecast.py rename to tests/unit/test_forecast.py diff --git a/tests/test_forecast_no_ts.py b/tests/unit/test_forecast_no_ts.py similarity index 100% rename from tests/test_forecast_no_ts.py rename to tests/unit/test_forecast_no_ts.py diff --git a/tests/test_generate_forecast.py b/tests/unit/test_generate_forecast.py similarity index 100% rename from tests/test_generate_forecast.py rename to tests/unit/test_generate_forecast.py diff --git a/tests/utils/test_file_path.py b/tests/unit/utils/test_file_path.py similarity index 100% rename from tests/utils/test_file_path.py rename to tests/unit/utils/test_file_path.py From 2a5b9fb05fabe783cac58260241d07fe63fd4c0b Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Mon, 18 Nov 2024 20:31:31 +0000 Subject: [PATCH 02/26] set test dir in step 1 --- .github/workflows/pytest.yaml | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 40dfe4c1..3c5e8a7b 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -10,13 +10,24 @@ on: types: [opened, synchronize, reopened, ready_for_review] jobs: call-run-python-tests: - uses: openclimatefix/.github/.github/workflows/python-test.yml@issue/pip-all - with: - # pytest-cov looks at this folder - pytest_cov_dir: "quartz_solar_forecast" - os_list: '["ubuntu-latest"]' - python-version: "['3.11']" - extra_commands: echo "HF_TOKEN=${{ vars.HF_TOKEN }}" > .env - pytest_numcpus: '1' - # if branch is 'main' then run all tests, if not only run unittests - test_dir: ${{ github.ref == 'refs/heads/main' && '' || 'tests/unit' }} + steps: + - name: Set test dir + run: | + export $TEST_DIR="tests/unit" + # if branch is 'main', or the repo is 'openclimatefix/quartz-solar-forecast' + # then run all tests, else run only unit tests + if [ ${{ github.event.repository.name }} == 'openclimatefix/quartz-solar-forecast' ] || [ ${{ github.ref }} == 'refs/heads/main' ]; then + export $TEST_DIR="" + fi + echo $TEST_DIR" + + - name: Run Python tests + uses: openclimatefix/.github/.github/workflows/python-test.yml@issue/pip-all + with: + # pytest-cov looks at this folder + pytest_cov_dir: "quartz_solar_forecast" + os_list: '["ubuntu-latest"]' + python-version: "['3.11']" + extra_commands: echo "HF_TOKEN=${{ vars.HF_TOKEN }}" > .env + pytest_numcpus: '1' + test_dir: $TEST_DIR \ No newline at end of file From c4bf3a8f23b3cdde4a47d5fb3d4523885d95a69e Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Mon, 18 Nov 2024 20:38:56 +0000 Subject: [PATCH 03/26] add runs-on --- .github/workflows/pytest.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 3c5e8a7b..f6e4ef13 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -8,8 +8,10 @@ on: - cron: "0 12 * * 1" pull_request_target: types: [opened, synchronize, reopened, ready_for_review] + jobs: call-run-python-tests: + runs-on: ubuntu-latest steps: - name: Set test dir run: | From ec8490ca1b23c0ee5fcdde22509cfedfd7f33d07 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Mon, 18 Nov 2024 20:46:12 +0000 Subject: [PATCH 04/26] divide out workflows --- .github/workflows/pytest.yaml | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index f6e4ef13..8f6206d9 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -10,10 +10,10 @@ on: types: [opened, synchronize, reopened, ready_for_review] jobs: - call-run-python-tests: + setup-test-dir: runs-on: ubuntu-latest steps: - - name: Set test dir + - name: Setup test dir run: | export $TEST_DIR="tests/unit" # if branch is 'main', or the repo is 'openclimatefix/quartz-solar-forecast' @@ -22,14 +22,16 @@ jobs: export $TEST_DIR="" fi echo $TEST_DIR" - - - name: Run Python tests - uses: openclimatefix/.github/.github/workflows/python-test.yml@issue/pip-all - with: - # pytest-cov looks at this folder - pytest_cov_dir: "quartz_solar_forecast" - os_list: '["ubuntu-latest"]' - python-version: "['3.11']" - extra_commands: echo "HF_TOKEN=${{ vars.HF_TOKEN }}" > .env - pytest_numcpus: '1' - test_dir: $TEST_DIR \ No newline at end of file + + call-run-python-tests: + - name: Run Python tests + needs: setup-test-dir + uses: openclimatefix/.github/.github/workflows/python-test.yml@issue/pip-all + with: + # pytest-cov looks at this folder + pytest_cov_dir: "quartz_solar_forecast" + os_list: '["ubuntu-latest"]' + python-version: "['3.11']" + extra_commands: echo "HF_TOKEN=${{ vars.HF_TOKEN }}" > .env + pytest_numcpus: '1' + test_dir: $TEST_DIR \ No newline at end of file From 9691a3efd9d76d4355719a7806bcd8a820b55ace Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Mon, 18 Nov 2024 20:47:41 +0000 Subject: [PATCH 05/26] format --- .github/workflows/pytest.yaml | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 8f6206d9..50bf5ef2 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -24,14 +24,13 @@ jobs: echo $TEST_DIR" call-run-python-tests: - - name: Run Python tests - needs: setup-test-dir - uses: openclimatefix/.github/.github/workflows/python-test.yml@issue/pip-all - with: - # pytest-cov looks at this folder - pytest_cov_dir: "quartz_solar_forecast" - os_list: '["ubuntu-latest"]' - python-version: "['3.11']" - extra_commands: echo "HF_TOKEN=${{ vars.HF_TOKEN }}" > .env - pytest_numcpus: '1' - test_dir: $TEST_DIR \ No newline at end of file + needs: setup-test-dir + uses: openclimatefix/.github/.github/workflows/python-test.yml@issue/pip-all + with: + # pytest-cov looks at this folder + pytest_cov_dir: "quartz_solar_forecast" + os_list: '["ubuntu-latest"]' + python-version: "['3.11']" + extra_commands: echo "HF_TOKEN=${{ vars.HF_TOKEN }}" > .env + pytest_numcpus: '1' + test_dir: $TEST_DIR \ No newline at end of file From 16d51abb3b30d567c2f776c20c346163ee756f86 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Mon, 18 Nov 2024 20:48:58 +0000 Subject: [PATCH 06/26] tidy --- .github/workflows/pytest.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 50bf5ef2..9ba480a7 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -15,11 +15,11 @@ jobs: steps: - name: Setup test dir run: | - export $TEST_DIR="tests/unit" + export TEST_DIR="tests/unit" # if branch is 'main', or the repo is 'openclimatefix/quartz-solar-forecast' # then run all tests, else run only unit tests if [ ${{ github.event.repository.name }} == 'openclimatefix/quartz-solar-forecast' ] || [ ${{ github.ref }} == 'refs/heads/main' ]; then - export $TEST_DIR="" + export TEST_DIR="" fi echo $TEST_DIR" From edf02a047f9701aea03276517b76988996c7c8d3 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Mon, 18 Nov 2024 20:50:25 +0000 Subject: [PATCH 07/26] fix --- .github/workflows/pytest.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 9ba480a7..e39b5a53 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -21,7 +21,7 @@ jobs: if [ ${{ github.event.repository.name }} == 'openclimatefix/quartz-solar-forecast' ] || [ ${{ github.ref }} == 'refs/heads/main' ]; then export TEST_DIR="" fi - echo $TEST_DIR" + echo $TEST_DIR call-run-python-tests: needs: setup-test-dir From d3ea54b2e46c19b94f6ba1cf6ed72c9d65b8d805 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Mon, 18 Nov 2024 20:53:44 +0000 Subject: [PATCH 08/26] use tests dir --- .github/workflows/pytest.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index e39b5a53..3f4e0d56 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -19,7 +19,7 @@ jobs: # if branch is 'main', or the repo is 'openclimatefix/quartz-solar-forecast' # then run all tests, else run only unit tests if [ ${{ github.event.repository.name }} == 'openclimatefix/quartz-solar-forecast' ] || [ ${{ github.ref }} == 'refs/heads/main' ]; then - export TEST_DIR="" + export TEST_DIR="tests" fi echo $TEST_DIR From 7da6ce11dab9d54df796f66dd0fe36758102fce8 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Mon, 18 Nov 2024 21:02:47 +0000 Subject: [PATCH 09/26] add comment --- .github/workflows/pytest.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 3f4e0d56..03300707 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -21,6 +21,7 @@ jobs: if [ ${{ github.event.repository.name }} == 'openclimatefix/quartz-solar-forecast' ] || [ ${{ github.ref }} == 'refs/heads/main' ]; then export TEST_DIR="tests" fi + # print out so we can see echo $TEST_DIR call-run-python-tests: From 095c669d35fa62136dd3d376a8e4ecee07b93d40 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Mon, 18 Nov 2024 21:04:31 +0000 Subject: [PATCH 10/26] change check to repo owner --- .github/workflows/pytest.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 03300707..578125a8 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -18,7 +18,7 @@ jobs: export TEST_DIR="tests/unit" # if branch is 'main', or the repo is 'openclimatefix/quartz-solar-forecast' # then run all tests, else run only unit tests - if [ ${{ github.event.repository.name }} == 'openclimatefix/quartz-solar-forecast' ] || [ ${{ github.ref }} == 'refs/heads/main' ]; then + if [[ ${{ github.repository_owner }} == 'openclimatefix' ]] || [[ ${{ github.ref }} == 'refs/heads/main' ]]; then export TEST_DIR="tests" fi # print out so we can see From 1b6bae9dd68e2b2caa0ee945dd0c61ef362b6d7e Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Mon, 18 Nov 2024 21:05:55 +0000 Subject: [PATCH 11/26] tidy --- .github/workflows/pytest.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 578125a8..deefd7c5 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -16,9 +16,8 @@ jobs: - name: Setup test dir run: | export TEST_DIR="tests/unit" - # if branch is 'main', or the repo is 'openclimatefix/quartz-solar-forecast' - # then run all tests, else run only unit tests - if [[ ${{ github.repository_owner }} == 'openclimatefix' ]] || [[ ${{ github.ref }} == 'refs/heads/main' ]]; then + # if branch is 'main', or the repo owner is 'openclimatefix' + if [[ ${{ github.repository_owner }} == 'openclimatefix' ]]; then export TEST_DIR="tests" fi # print out so we can see From 09733dd27a7895e54d2bd97d46ab71149b70b498 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Mon, 18 Nov 2024 21:08:43 +0000 Subject: [PATCH 12/26] change to !- pull request --- .github/workflows/pytest.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index deefd7c5..c6346646 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -17,7 +17,7 @@ jobs: run: | export TEST_DIR="tests/unit" # if branch is 'main', or the repo owner is 'openclimatefix' - if [[ ${{ github.repository_owner }} == 'openclimatefix' ]]; then + if [[ ${{ github.event_name }} != 'pull_request' ]]; then export TEST_DIR="tests" fi # print out so we can see From 4072c97acce38eff72d39b912d5daa943f1bb384 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Mon, 18 Nov 2024 21:09:55 +0000 Subject: [PATCH 13/26] update comment --- .github/workflows/pytest.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index c6346646..2f60e53f 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -16,7 +16,7 @@ jobs: - name: Setup test dir run: | export TEST_DIR="tests/unit" - # if branch is 'main', or the repo owner is 'openclimatefix' + # if github action is not a pull request, then run all tests if [[ ${{ github.event_name }} != 'pull_request' ]]; then export TEST_DIR="tests" fi From 0d348e94eaa3325113e19cef497f2558f42758a8 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Mon, 18 Nov 2024 21:18:22 +0000 Subject: [PATCH 14/26] echo --- .github/workflows/pytest.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 2f60e53f..86749df6 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -33,4 +33,4 @@ jobs: python-version: "['3.11']" extra_commands: echo "HF_TOKEN=${{ vars.HF_TOKEN }}" > .env pytest_numcpus: '1' - test_dir: $TEST_DIR \ No newline at end of file + test_dir: echo $TEST_DIR \ No newline at end of file From 38373333c34ef6d90ee474c1c4215757f87d8d25 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Mon, 18 Nov 2024 21:22:57 +0000 Subject: [PATCH 15/26] use github output --- .github/workflows/pytest.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 86749df6..acabc94e 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -22,6 +22,7 @@ jobs: fi # print out so we can see echo $TEST_DIR + echo "test_dir=$TEST_DIR" >> $GITHUB_OUTPUT call-run-python-tests: needs: setup-test-dir @@ -33,4 +34,4 @@ jobs: python-version: "['3.11']" extra_commands: echo "HF_TOKEN=${{ vars.HF_TOKEN }}" > .env pytest_numcpus: '1' - test_dir: echo $TEST_DIR \ No newline at end of file + test_dir: ${{ needs.setup-test-dir.outputs.test_dir }} \ No newline at end of file From 7c2475de80393c0847e7a8409bf26996eae18645 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Mon, 18 Nov 2024 21:28:55 +0000 Subject: [PATCH 16/26] run second job if push --- .github/workflows/pytest.yaml | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index acabc94e..e8a479a4 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -10,22 +10,7 @@ on: types: [opened, synchronize, reopened, ready_for_review] jobs: - setup-test-dir: - runs-on: ubuntu-latest - steps: - - name: Setup test dir - run: | - export TEST_DIR="tests/unit" - # if github action is not a pull request, then run all tests - if [[ ${{ github.event_name }} != 'pull_request' ]]; then - export TEST_DIR="tests" - fi - # print out so we can see - echo $TEST_DIR - echo "test_dir=$TEST_DIR" >> $GITHUB_OUTPUT - - call-run-python-tests: - needs: setup-test-dir + call-run-python-tests-unit: uses: openclimatefix/.github/.github/workflows/python-test.yml@issue/pip-all with: # pytest-cov looks at this folder @@ -34,4 +19,17 @@ jobs: python-version: "['3.11']" extra_commands: echo "HF_TOKEN=${{ vars.HF_TOKEN }}" > .env pytest_numcpus: '1' - test_dir: ${{ needs.setup-test-dir.outputs.test_dir }} \ No newline at end of file + test_dir: tests/unit + + call-run-python-tests-all: + # only run on push, not external PR + uses: openclimatefix/.github/.github/workflows/python-test.yml@issue/pip-all + if: github.event_name == 'push' + with: + # pytest-cov looks at this folder + pytest_cov_dir: "quartz_solar_forecast" + os_list: '["ubuntu-latest"]' + python-version: "['3.11']" + extra_commands: echo "HF_TOKEN=${{ vars.HF_TOKEN }}" > .env + pytest_numcpus: '1' + test_dir: tests \ No newline at end of file From 28b5674964fc505e5aaedb9e5e6a24ab192186df Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Mon, 18 Nov 2024 21:29:30 +0000 Subject: [PATCH 17/26] format --- .github/workflows/pytest.yaml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index e8a479a4..49133f01 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -21,15 +21,15 @@ jobs: pytest_numcpus: '1' test_dir: tests/unit - call-run-python-tests-all: - # only run on push, not external PR - uses: openclimatefix/.github/.github/workflows/python-test.yml@issue/pip-all - if: github.event_name == 'push' - with: - # pytest-cov looks at this folder - pytest_cov_dir: "quartz_solar_forecast" - os_list: '["ubuntu-latest"]' - python-version: "['3.11']" - extra_commands: echo "HF_TOKEN=${{ vars.HF_TOKEN }}" > .env - pytest_numcpus: '1' - test_dir: tests \ No newline at end of file + call-run-python-tests-all: + # only run on push, not external PR + uses: openclimatefix/.github/.github/workflows/python-test.yml@issue/pip-all + if: github.event_name == 'push' + with: + # pytest-cov looks at this folder + pytest_cov_dir: "quartz_solar_forecast" + os_list: '["ubuntu-latest"]' + python-version: "['3.11']" + extra_commands: echo "HF_TOKEN=${{ vars.HF_TOKEN }}" > .env + pytest_numcpus: '1' + test_dir: tests \ No newline at end of file From 14530d9d745e4b3375ef4553e20777a2204eee5a Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Mon, 18 Nov 2024 21:34:59 +0000 Subject: [PATCH 18/26] pytest.mark.integration --- tests/integration/eval/test_evaluation.py | 2 +- tests/integration/eval/test_nwp.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/eval/test_evaluation.py b/tests/integration/eval/test_evaluation.py index ce94986d..9fb42730 100644 --- a/tests/integration/eval/test_evaluation.py +++ b/tests/integration/eval/test_evaluation.py @@ -4,7 +4,7 @@ import pytest -@pytest.test.integration +@pytest.mark.integration def test_run_eval(): # create a fake dataframe diff --git a/tests/integration/eval/test_nwp.py b/tests/integration/eval/test_nwp.py index 46b786cc..0b18f18f 100644 --- a/tests/integration/eval/test_nwp.py +++ b/tests/integration/eval/test_nwp.py @@ -4,7 +4,7 @@ # can take ~ 1 minute to run -@pytest.test.integration +@pytest.mark.integration def test_get_nwp(): # make test dataset file test_set_df = pd.DataFrame( From f1ff28e71cabea235edce3296bafc0c46f663b39 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Tue, 19 Nov 2024 08:49:13 +0000 Subject: [PATCH 19/26] add timeout --- quartz_solar_forecast/weather/open_meteo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quartz_solar_forecast/weather/open_meteo.py b/quartz_solar_forecast/weather/open_meteo.py index cc6b44f4..b3207938 100644 --- a/quartz_solar_forecast/weather/open_meteo.py +++ b/quartz_solar_forecast/weather/open_meteo.py @@ -150,7 +150,7 @@ def get_hourly_weather( ] url = self._build_url(latitude, longitude, start_date, end_date, variables) try: - response = requests.get(url) + response = requests.get(url, timeout=5) except requests.exceptions.Timeout: raise TimeoutError(f"Request to OpenMeteo API timed out. URl - {url}") From 822c1d46aac6751ca038ae8e44db3f6bf0b3c983 Mon Sep 17 00:00:00 2001 From: Megawattz Date: Fri, 24 Jan 2025 09:10:40 +0100 Subject: [PATCH 20/26] remove timeout from merge --- quartz_solar_forecast/weather/open_meteo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quartz_solar_forecast/weather/open_meteo.py b/quartz_solar_forecast/weather/open_meteo.py index b382253c..7da71108 100644 --- a/quartz_solar_forecast/weather/open_meteo.py +++ b/quartz_solar_forecast/weather/open_meteo.py @@ -155,7 +155,7 @@ def get_hourly_weather( retry_session = retry(cache_session, retries=5, backoff_factor=0.2) try: openmeteo = openmeteo_requests.Client(session=retry_session) - response = openmeteo.weather_api(url, params={}, timeout=5) + response = openmeteo.weather_api(url, params={}) except requests.exceptions.Timeout: raise TimeoutError(f"Request to OpenMeteo API timed out. URl - {url}") From 9bd80f534ea10a04e23db4410378a26cc3037961 Mon Sep 17 00:00:00 2001 From: Megawattz Date: Fri, 24 Jan 2025 11:03:48 +0100 Subject: [PATCH 21/26] turn normalised error into percent --- quartz_solar_forecast/eval/metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quartz_solar_forecast/eval/metrics.py b/quartz_solar_forecast/eval/metrics.py index fedd2220..8497322d 100644 --- a/quartz_solar_forecast/eval/metrics.py +++ b/quartz_solar_forecast/eval/metrics.py @@ -35,7 +35,7 @@ def metrics(results_df: pd.DataFrame, pv_metadata: pd.DataFrame, include_night: .mean(), 4, ) - print(f"MAE: {mae} kw, normalized {mae_normalized} %") + print(f"MAE: {mae} kw, normalized {100*mae_normalized} %") # calculate metrics over the different horizons hours # find all unique horizon_hours From 6d2f7f1cafb5f8937ced66beb0f1f61fc4315e15 Mon Sep 17 00:00:00 2001 From: Megawattz Date: Fri, 24 Jan 2025 11:10:59 +0100 Subject: [PATCH 22/26] limit metrics to 3sf --- quartz_solar_forecast/eval/metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quartz_solar_forecast/eval/metrics.py b/quartz_solar_forecast/eval/metrics.py index 8497322d..ea79ce2b 100644 --- a/quartz_solar_forecast/eval/metrics.py +++ b/quartz_solar_forecast/eval/metrics.py @@ -74,7 +74,7 @@ def metrics(results_df: pd.DataFrame, pv_metadata: pd.DataFrame, include_night: ) print( - f"MAE for horizon {horizon_group}: {mae} +- {1.96*sem}. mae_normalized: {100*mae_normalized} %" + f"MAE for horizon {horizon_group}: {mae} +- {1.96*sem:.3g}. mae_normalized: {100*mae_normalized:.3g} %" ) # TODO add more metrics using ocf_ml_metrics From 54a0bff20ab849ac875bad38451ae4c93f4e026c Mon Sep 17 00:00:00 2001 From: Megawattz Date: Fri, 24 Jan 2025 11:12:13 +0100 Subject: [PATCH 23/26] update site used in test_evaluation script to give non nan result --- tests/integration/eval/test_evaluation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/eval/test_evaluation.py b/tests/integration/eval/test_evaluation.py index 9fb42730..9a9cbb58 100644 --- a/tests/integration/eval/test_evaluation.py +++ b/tests/integration/eval/test_evaluation.py @@ -17,7 +17,7 @@ def test_run_eval(): "pv_id", "timestamp", ], - data=[[8215, "2021-01-26 01:15:00"], [8215, "2021-01-30 16:30:00"]], + data=[[7593, "2021-08-21 12:00:00"], [7593, "2021-10-04 20:00:00"]], ) testset_filename = tmpdirname + "/test_dataset.csv" From 8ea97ba4259b9e18abd4982d704d01b8ca697241 Mon Sep 17 00:00:00 2001 From: Megawattz Date: Fri, 24 Jan 2025 11:14:02 +0100 Subject: [PATCH 24/26] add pytest.ini file to mark integration tests --- pytest.ini | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 pytest.ini diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..7c95c298 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +markers = + integration: marks tests as integration tests \ No newline at end of file From df662f33a61cc19f0a46ee86664d3cf54743ded2 Mon Sep 17 00:00:00 2001 From: Megawattz Date: Fri, 24 Jan 2025 12:02:44 +0100 Subject: [PATCH 25/26] update forecast test with no ts --- tests/unit/test_forecast_no_ts.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/unit/test_forecast_no_ts.py b/tests/unit/test_forecast_no_ts.py index 0450379d..a66b7b1c 100644 --- a/tests/unit/test_forecast_no_ts.py +++ b/tests/unit/test_forecast_no_ts.py @@ -7,22 +7,21 @@ def test_run_forecast_no_ts(): # make input data site = PVSite(latitude=51.75, longitude=-1.25, capacity_kwp=1.25) - current_ts = pd.Timestamp.now().round("15min") - current_hr = pd.Timestamp.now().round(freq='h') + current_ts = pd.Timestamp.now() # run gradient boosting model with no ts - predications_df = run_forecast(site=site, model="gb", ts=current_ts) + predications_df = run_forecast(site=site, model="gb") # check current ts agrees with dataset - assert predications_df.index.min() == current_ts + assert predications_df.index.min() >= current_ts - pd.Timedelta(hours=1) print(predications_df) print(f"Current time: {current_ts}") print(f"Max: {predications_df['power_kw'].max()}") # run xgb model with no ts - predications_df = run_forecast(site=site, model="xgb", ts=current_ts) + predications_df = run_forecast(site=site, model="xgb") # check current ts agrees with dataset - assert predications_df.index.min() == current_hr + assert predications_df.index.min() >= current_ts - pd.Timedelta(hours=1) print(predications_df) print(f"Current time: {current_ts}") From 6786a4c4ed8b6c7afc8d46282a085667be5e722e Mon Sep 17 00:00:00 2001 From: Megawattz Date: Fri, 24 Jan 2025 12:06:10 +0100 Subject: [PATCH 26/26] update forecast test with no ts spelling --- tests/unit/test_forecast_no_ts.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/unit/test_forecast_no_ts.py b/tests/unit/test_forecast_no_ts.py index a66b7b1c..0df021b6 100644 --- a/tests/unit/test_forecast_no_ts.py +++ b/tests/unit/test_forecast_no_ts.py @@ -10,19 +10,19 @@ def test_run_forecast_no_ts(): current_ts = pd.Timestamp.now() # run gradient boosting model with no ts - predications_df = run_forecast(site=site, model="gb") + predictions_df = run_forecast(site=site, model="gb") # check current ts agrees with dataset - assert predications_df.index.min() >= current_ts - pd.Timedelta(hours=1) + assert predictions_df.index.min() >= current_ts - pd.Timedelta(hours=1) - print(predications_df) + print(predictions_df) print(f"Current time: {current_ts}") - print(f"Max: {predications_df['power_kw'].max()}") + print(f"Max: {predictions_df['power_kw'].max()}") # run xgb model with no ts - predications_df = run_forecast(site=site, model="xgb") + predictions_df = run_forecast(site=site, model="xgb") # check current ts agrees with dataset - assert predications_df.index.min() >= current_ts - pd.Timedelta(hours=1) + assert predictions_df.index.min() >= current_ts - pd.Timedelta(hours=1) - print(predications_df) + print(predictions_df) print(f"Current time: {current_ts}") - print(f"Max: {predications_df['power_kw'].max()}") \ No newline at end of file + print(f"Max: {predictions_df['power_kw'].max()}") \ No newline at end of file