diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 01a8113c5..da7d19d11 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,7 @@ jobs: image: postgres:latest env: POSTGRES_PASSWORD: postgres + POSTGRES_DB: test_database_name ports: - 5432:5432 # needed because the postgres container does not provide a healthcheck @@ -40,11 +41,22 @@ jobs: - name: Fetch stats run: | mkdir out - ./mock_stats_for_unit_tests.sh - - name: Run tests + ./partial_stats_for_unit_tests.sh + - name: Set up django + env: + DATABASE_URL: "postgres://postgres:postgres@localhost:5432/test_database_name" + run: | + python manage.py collectstatic + python manage.py migrate + python manage.py dashboard_import + - name: Run non-django tests env: DATABASE_URL: "postgres://postgres:postgres@localhost:5432/test_database_name" run: pytest --cov . + - name: Run django tests + env: + DATABASE_URL: "postgres://postgres:postgres@localhost:5432/test_database_name" + run: python manage.py test # - name: Coveralls # run: coveralls --service=github-actions # env: diff --git a/iati_dashboard/templates/element.html b/iati_dashboard/templates/element.html index 2cb713ba3..e1c7f36c0 100644 --- a/iati_dashboard/templates/element.html +++ b/iati_dashboard/templates/element.html @@ -33,6 +33,7 @@

Publishing this {{ element_or_attribu {% for publisher in func.sorted(publishers) %} + {% if publisher in current_stats.inverted_publisher.activities %} {{ publisher }} {% with publisher_inverted=func.get_publisher_stats(publisher, 'inverted-file') %} @@ -45,6 +46,7 @@

Publishing this {{ element_or_attribu {% endwith %} {{ current_stats.inverted_publisher.activities[publisher] }} + {% endif %} {% endfor %} diff --git a/iati_dashboard/ui/tests.py b/iati_dashboard/ui/tests.py index cffb7b1ab..fcdc7f8d5 100644 --- a/iati_dashboard/ui/tests.py +++ b/iati_dashboard/ui/tests.py @@ -14,55 +14,38 @@ def test_top_pages(self): """Test the index and top hierarchy pages return a 200 status code""" self.assertEqual(self.client.get(reverse("dash-index")).status_code, 200) - self.assertEqual(self.client.get(reverse("dash-headlines")).status_code, 200) - self.assertEqual(self.client.get(reverse("dash-dataquality")).status_code, 200) - self.assertEqual(self.client.get(reverse("dash-publishingstats")).status_code, 200) - self.assertEqual(self.client.get(reverse("dash-exploringdata")).status_code, 200) - self.assertEqual(self.client.get(reverse("dash-faq")).status_code, 200) - self.assertEqual(self.client.get(reverse("dash-registrationagencies")).status_code, 200) - - def test_headlines(self): - """Test the headlines pages""" - - self.assertEqual(self.client.get(reverse("dash-headlines-files")).status_code, 200) - self.assertEqual(self.client.get(reverse("dash-headlines-activities")).status_code, 200) self.assertEqual(self.client.get(reverse("dash-headlines-publishers")).status_code, 200) - self.assertEqual(self.client.get(reverse("dash-headlines-publisher-detail", args=("undp",))).status_code, 200) + self.assertEqual(self.client.get(reverse("dash-headlines-publisher-detail", args=("zsl",))).status_code, 200) self.assertEqual( self.client.get(reverse("dash-headlines-publisher-detail", args=("not-a-valid-publisher",))).status_code, 404, ) + self.assertEqual(self.client.get(reverse("dash-errors")).status_code, 200) + self.assertEqual(self.client.get(reverse("dash-publishingstats")).status_code, 200) + self.assertEqual(self.client.get(reverse("dash-exploringdata")).status_code, 200) + self.assertEqual(self.client.get(reverse("dash-faq")).status_code, 200) + self.assertEqual(self.client.get(reverse("dash-registrationagencies")).status_code, 200) - def test_dataquality(self): + def test_errors(self): """Test the data quality pages""" - self.assertEqual(self.client.get(reverse("dash-dataquality-download")).status_code, 200) - self.assertEqual(self.client.get(reverse("dash-dataquality-download-json")).status_code, 200) - self.assertEqual(self.client.get(reverse("dash-dataquality-xml")).status_code, 200) - self.assertEqual(self.client.get(reverse("dash-dataquality-validation")).status_code, 200) - self.assertEqual(self.client.get(reverse("dash-dataquality-versions")).status_code, 200) - self.assertEqual(self.client.get(reverse("dash-dataquality-organisation")).status_code, 200) - self.assertEqual(self.client.get(reverse("dash-dataquality-identifiers")).status_code, 200) - self.assertEqual(self.client.get(reverse("dash-dataquality-reportingorgs")).status_code, 200) - self.assertEqual(self.client.get(reverse("dash-dataquality-licenses")).status_code, 200) - self.assertEqual( - self.client.get(reverse("dash-dataquality-licenses-detail", args=("cc-by",))).status_code, 200 - ) - self.assertEqual( - self.client.get(reverse("dash-dataquality-licenses-detail", args=("not-a-valid-license",))).status_code, - 404, - ) + self.assertEqual(self.client.get(reverse("dash-errors-download")).status_code, 200) + self.assertEqual(self.client.get(reverse("dash-errors-download-json")).status_code, 200) + self.assertEqual(self.client.get(reverse("dash-errors-xml")).status_code, 200) + self.assertEqual(self.client.get(reverse("dash-errors-validation")).status_code, 200) + self.assertEqual(self.client.get(reverse("dash-identifiers")).status_code, 200) + self.assertEqual(self.client.get(reverse("dash-errors-reportingorgs")).status_code, 200) def test_publishingstats_timeliness(self): """Test timeliness pages in the publishing statistics section""" - self.assertEqual(self.client.get(reverse("dash-publishingstats-timeliness")).status_code, 200) + self.assertEqual(self.client.get(reverse("dash-publishingstats-timeliness-frequency")).status_code, 200) self.assertEqual(self.client.get(reverse("dash-publishingstats-timeliness-timelag")).status_code, 200) def test_publishingstats_comprehensiveness(self): """Test comprehensiveness pages in the publishing statistics section""" - self.assertEqual(self.client.get(reverse("dash-publishingstats-comprehensiveness")).status_code, 200) + self.assertEqual(self.client.get(reverse("dash-publishingstats-comprehensiveness-summary")).status_code, 200) self.assertEqual(self.client.get(reverse("dash-publishingstats-comprehensiveness-core")).status_code, 200) self.assertEqual( self.client.get(reverse("dash-publishingstats-comprehensiveness-financials")).status_code, 200 @@ -88,6 +71,8 @@ def test_publishingstats_humanitarian(self): def test_exploringdata(self): """Test the exploring data pages""" + self.assertEqual(self.client.get(reverse("dash-headlines-files")).status_code, 200) + self.assertEqual(self.client.get(reverse("dash-headlines-activities")).status_code, 200) self.assertEqual(self.client.get(reverse("dash-exploringdata-booleans")).status_code, 200) self.assertEqual(self.client.get(reverse("dash-exploringdata-codelists")).status_code, 200) self.assertEqual( @@ -127,10 +112,18 @@ def test_exploringdata(self): 404, ) self.assertEqual(self.client.get(reverse("dash-exploringdata-dates")).status_code, 200) + self.assertEqual(self.client.get(reverse("dash-versions")).status_code, 200) + self.assertEqual(self.client.get(reverse("dash-organisation")).status_code, 200) + self.assertEqual(self.client.get(reverse("dash-licenses")).status_code, 200) + self.assertEqual(self.client.get(reverse("dash-licenses-detail", args=("cc-by",))).status_code, 200) + self.assertEqual( + self.client.get(reverse("dash-licenses-detail", args=("not-a-valid-license",))).status_code, + 404, + ) self.assertEqual(self.client.get(reverse("dash-exploringdata-elements")).status_code, 200) self.assertEqual( self.client.get( - reverse("dash-exploringdata-elements-detail", args=("iati-activity_activity-date_narrative",)) + reverse("dash-exploringdata-elements-detail", args=("iati-activity_activity-date_@iso-date",)) ).status_code, 200, ) @@ -172,7 +165,7 @@ def test_headlines_and_misc(self): self._url_and_view_helper( { "index": "dash-index", - "headlines": "dash-headlines", + "headlines": "dash-index", "files": "dash-headlines-files", "activities": "dash-headlines-activities", "publishers": "dash-headlines-publishers", @@ -181,7 +174,7 @@ def test_headlines_and_misc(self): } ) - def test_dataquality(self): + def test_errors(self): """Test data quality pages redirect to their new locations""" # This is not particularly DRY as a similar dictionary is created in views.py @@ -189,15 +182,15 @@ def test_dataquality(self): # refactor of what goes into the context. self._url_and_view_helper( { - "data_quality": "dash-dataquality", - "download": "dash-dataquality-download", - "xml": "dash-dataquality-xml", - "validation": "dash-dataquality-validation", - "versions": "dash-dataquality-versions", - "organisation": "dash-dataquality-organisation", - "identifiers": "dash-dataquality-identifiers", - "reporting_orgs": "dash-dataquality-reportingorgs", - "licenses": "dash-dataquality-licenses", + "data_quality": "dash-errors", + "download": "dash-errors-download", + "xml": "dash-errors-xml", + "validation": "dash-errors-validation", + "versions": "dash-versions", + "organisation": "dash-organisation", + "identifiers": "dash-identifiers", + "reporting_orgs": "dash-errors-reportingorgs", + "licenses": "dash-licenses", } ) @@ -210,10 +203,10 @@ def test_publishingstats(self): self._url_and_view_helper( { "publishing_stats": "dash-publishingstats", - "timeliness": "dash-publishingstats-timeliness", + "timeliness": "dash-publishingstats-timeliness-frequency", "timeliness_timelag": "dash-publishingstats-timeliness-timelag", "forwardlooking": "dash-publishingstats-forwardlooking", - "comprehensiveness": "dash-publishingstats-comprehensiveness", + "comprehensiveness": "dash-publishingstats-comprehensiveness-summary", "comprehensiveness_core": "dash-publishingstats-comprehensiveness-core", "comprehensiveness_financials": "dash-publishingstats-comprehensiveness-financials", "comprehensiveness_valueadded": "dash-publishingstats-comprehensiveness-valueadded", @@ -244,16 +237,14 @@ def test_slug_page_redirects(self): """Test pages with slugs redirect to the section page""" self.assertRedirects( - self.client.get(r"/publisher/undp.html"), reverse("dash-headlines-publishers"), status_code=301 - ) - self.assertRedirects( - self.client.get(r"/license/cc-by.html"), reverse("dash-dataquality-licenses"), status_code=301 + self.client.get(r"/publisher/zsl.html"), reverse("dash-headlines-publishers"), status_code=301 ) + self.assertRedirects(self.client.get(r"/license/cc-by.html"), reverse("dash-licenses"), status_code=301) self.assertRedirects( self.client.get(r"/codelist/2/budget_@type.html"), reverse("dash-exploringdata-codelists"), status_code=301 ) self.assertRedirects( - self.client.get(r"/element/iati-activity_activity-date_narrative.html"), + self.client.get(r"/element/iati-activity_activity-date_@iso-date.html"), reverse("dash-exploringdata-elements"), status_code=301, ) diff --git a/iati_dashboard/ui/urls.py b/iati_dashboard/ui/urls.py index b6898ba25..60d16e125 100644 --- a/iati_dashboard/ui/urls.py +++ b/iati_dashboard/ui/urls.py @@ -130,7 +130,8 @@ ), # Redirects to support any users with bookmarks to pages on the old Dashboard. path("index.html", RedirectView.as_view(pattern_name="dash-index", permanent=True)), - path("headlines.html", RedirectView.as_view(pattern_name="dash-headlines", permanent=True)), + # We've remove the headlines page, so redirect to the index + path("headlines.html", RedirectView.as_view(pattern_name="dash-index", permanent=True)), path("data_quality.html", RedirectView.as_view(pattern_name="dash-errors", permanent=True)), path("exploring_data.html", RedirectView.as_view(pattern_name="dash-exploringdata-elements", permanent=True)), path("publishers.html", RedirectView.as_view(pattern_name="dash-headlines-publishers", permanent=True)), diff --git a/iati_dashboard/ui/views.py b/iati_dashboard/ui/views.py index 81db14f66..183de776a 100644 --- a/iati_dashboard/ui/views.py +++ b/iati_dashboard/ui/views.py @@ -280,6 +280,9 @@ def headlines_files(request): def headlines_publisher_detail(request, publisher=None): + if publisher not in publisher_name: + raise Http404("Publisher does not exist") + template = loader.get_template("publisher.html") context = _make_context("publishers") diff --git a/mock_stats_for_unit_tests.sh b/mock_stats_for_unit_tests.sh deleted file mode 100755 index 26b6d0b10..000000000 --- a/mock_stats_for_unit_tests.sh +++ /dev/null @@ -1,9 +0,0 @@ -set -eux - -mkdir stats-calculated -curl --compressed "https://dev.merged.dashboard.iatistandard.org/stats/ckan.json" > stats-calculated/ckan.json -mkdir -p stats-calculated/current/aggregated-publisher -mkdir -p stats-calculated/current/inverted-publisher -for f in activities codelist_values_by_major_version elements; do - echo "{}" > stats-calculated/current/inverted-publisher/$f.json -done diff --git a/partial_stats_for_unit_tests.sh b/partial_stats_for_unit_tests.sh new file mode 100755 index 000000000..a419bebff --- /dev/null +++ b/partial_stats_for_unit_tests.sh @@ -0,0 +1,23 @@ +set -eux + +publisher_short_name=zsl + +if [ -d stats-calculated ]; then + echo "stats-calculated directory already exists, exiting." + exit +fi +wget -m --no-parent https://dev.merged.dashboard.iatistandard.org/stats/current/aggregated/ +wget -m --no-parent https://dev.merged.dashboard.iatistandard.org/stats/current/aggregated-publisher/$publisher_short_name/ +wget -m --no-parent https://dev.merged.dashboard.iatistandard.org/stats/current/aggregated-file/$publisher_short_name/ +wget -m --no-parent https://dev.merged.dashboard.iatistandard.org/stats/current/inverted-publisher/ +wget -m --no-parent https://dev.merged.dashboard.iatistandard.org/stats/current/inverted-file/ +wget -m --no-parent https://dev.merged.dashboard.iatistandard.org/stats/current/inverted-file-publisher/$publisher_short_name/ +wget -m --no-parent https://dev.merged.dashboard.iatistandard.org/stats/gitaggregate-publisher-dated/$publisher_short_name/ +mv dev.merged.dashboard.iatistandard.org/stats stats-calculated +for file in ckan.json licenses.json gitdate.json; do + curl --compressed https://dev.merged.dashboard.iatistandard.org/stats/$file > stats-calculated/$file +done + +cat stats-calculated/current/inverted-publisher/activities.json | jq "{$publisher_short_name: .$publisher_short_name}" > activities.json +mv activities.json stats-calculated/current/inverted-publisher/activities.json +rm stats-calculated/current/aggregated-publisher/$publisher_short_name/index.html*