Merge pull request #648 from IATI/fix-django-tests

Fix django tests and run them in CI
IATI · Jan 29, 2025 · dd01ca7 · dd01ca7
2 parents 9f74698 + 3d6a143
commit dd01ca7
Show file tree

Hide file tree

Showing 7 changed files with 86 additions and 63 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -8,6 +8,7 @@ jobs:
         image: postgres:latest
         env:
           POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: test_database_name
         ports:
           - 5432:5432
         # needed because the postgres container does not provide a healthcheck
@@ -40,11 +41,22 @@ jobs:
     - name: Fetch stats
       run: |
         mkdir out
-        ./mock_stats_for_unit_tests.sh
-    - name: Run tests
+        ./partial_stats_for_unit_tests.sh
+    - name: Set up django
+      env:
+        DATABASE_URL: "postgres://postgres:postgres@localhost:5432/test_database_name"
+      run: |
+        python manage.py collectstatic
+        python manage.py migrate
+        python manage.py dashboard_import
+    - name: Run non-django tests
       env:
         DATABASE_URL: "postgres://postgres:postgres@localhost:5432/test_database_name"
       run: pytest --cov .
+    - name: Run django tests
+      env:
+        DATABASE_URL: "postgres://postgres:postgres@localhost:5432/test_database_name"
+      run: python manage.py test
     # - name: Coveralls
     #   run: coveralls --service=github-actions
     #   env:

diff --git a/iati_dashboard/templates/element.html b/iati_dashboard/templates/element.html
@@ -33,6 +33,7 @@ <h3 class="dashboard-panel-heading__title">Publishing this {{ element_or_attribu
           </thead>
           <tbody>
             {% for publisher in func.sorted(publishers) %}
+            {% if publisher in current_stats.inverted_publisher.activities %}
             <tr>
                 <td><a href="{{ url('dash-headlines-publisher-detail', args=[publisher]) }}">{{ publisher }}</a></td>
                 {% with publisher_inverted=func.get_publisher_stats(publisher, 'inverted-file') %}
@@ -45,6 +46,7 @@ <h3 class="dashboard-panel-heading__title">Publishing this {{ element_or_attribu
                 {% endwith %}
                 <td>{{ current_stats.inverted_publisher.activities[publisher] }}</td>
             </tr>
+            {% endif %}
             {% endfor %}
           </tbody>
         </table>

diff --git a/iati_dashboard/ui/tests.py b/iati_dashboard/ui/tests.py
@@ -14,55 +14,38 @@ def test_top_pages(self):
         """Test the index and top hierarchy pages return a 200 status code"""
 
         self.assertEqual(self.client.get(reverse("dash-index")).status_code, 200)
-        self.assertEqual(self.client.get(reverse("dash-headlines")).status_code, 200)
-        self.assertEqual(self.client.get(reverse("dash-dataquality")).status_code, 200)
-        self.assertEqual(self.client.get(reverse("dash-publishingstats")).status_code, 200)
-        self.assertEqual(self.client.get(reverse("dash-exploringdata")).status_code, 200)
-        self.assertEqual(self.client.get(reverse("dash-faq")).status_code, 200)
-        self.assertEqual(self.client.get(reverse("dash-registrationagencies")).status_code, 200)
-
-    def test_headlines(self):
-        """Test the headlines pages"""
-
-        self.assertEqual(self.client.get(reverse("dash-headlines-files")).status_code, 200)
-        self.assertEqual(self.client.get(reverse("dash-headlines-activities")).status_code, 200)
         self.assertEqual(self.client.get(reverse("dash-headlines-publishers")).status_code, 200)
-        self.assertEqual(self.client.get(reverse("dash-headlines-publisher-detail", args=("undp",))).status_code, 200)
+        self.assertEqual(self.client.get(reverse("dash-headlines-publisher-detail", args=("zsl",))).status_code, 200)
         self.assertEqual(
             self.client.get(reverse("dash-headlines-publisher-detail", args=("not-a-valid-publisher",))).status_code,
             404,
         )
+        self.assertEqual(self.client.get(reverse("dash-errors")).status_code, 200)
+        self.assertEqual(self.client.get(reverse("dash-publishingstats")).status_code, 200)
+        self.assertEqual(self.client.get(reverse("dash-exploringdata")).status_code, 200)
+        self.assertEqual(self.client.get(reverse("dash-faq")).status_code, 200)
+        self.assertEqual(self.client.get(reverse("dash-registrationagencies")).status_code, 200)
 
-    def test_dataquality(self):
+    def test_errors(self):
         """Test the data quality pages"""
 
-        self.assertEqual(self.client.get(reverse("dash-dataquality-download")).status_code, 200)
-        self.assertEqual(self.client.get(reverse("dash-dataquality-download-json")).status_code, 200)
-        self.assertEqual(self.client.get(reverse("dash-dataquality-xml")).status_code, 200)
-        self.assertEqual(self.client.get(reverse("dash-dataquality-validation")).status_code, 200)
-        self.assertEqual(self.client.get(reverse("dash-dataquality-versions")).status_code, 200)
-        self.assertEqual(self.client.get(reverse("dash-dataquality-organisation")).status_code, 200)
-        self.assertEqual(self.client.get(reverse("dash-dataquality-identifiers")).status_code, 200)
-        self.assertEqual(self.client.get(reverse("dash-dataquality-reportingorgs")).status_code, 200)
-        self.assertEqual(self.client.get(reverse("dash-dataquality-licenses")).status_code, 200)
-        self.assertEqual(
-            self.client.get(reverse("dash-dataquality-licenses-detail", args=("cc-by",))).status_code, 200
-        )
-        self.assertEqual(
-            self.client.get(reverse("dash-dataquality-licenses-detail", args=("not-a-valid-license",))).status_code,
-            404,
-        )
+        self.assertEqual(self.client.get(reverse("dash-errors-download")).status_code, 200)
+        self.assertEqual(self.client.get(reverse("dash-errors-download-json")).status_code, 200)
+        self.assertEqual(self.client.get(reverse("dash-errors-xml")).status_code, 200)
+        self.assertEqual(self.client.get(reverse("dash-errors-validation")).status_code, 200)
+        self.assertEqual(self.client.get(reverse("dash-identifiers")).status_code, 200)
+        self.assertEqual(self.client.get(reverse("dash-errors-reportingorgs")).status_code, 200)
 
     def test_publishingstats_timeliness(self):
         """Test timeliness pages in the publishing statistics section"""
 
-        self.assertEqual(self.client.get(reverse("dash-publishingstats-timeliness")).status_code, 200)
+        self.assertEqual(self.client.get(reverse("dash-publishingstats-timeliness-frequency")).status_code, 200)
         self.assertEqual(self.client.get(reverse("dash-publishingstats-timeliness-timelag")).status_code, 200)
 
     def test_publishingstats_comprehensiveness(self):
         """Test comprehensiveness pages in the publishing statistics section"""
 
-        self.assertEqual(self.client.get(reverse("dash-publishingstats-comprehensiveness")).status_code, 200)
+        self.assertEqual(self.client.get(reverse("dash-publishingstats-comprehensiveness-summary")).status_code, 200)
         self.assertEqual(self.client.get(reverse("dash-publishingstats-comprehensiveness-core")).status_code, 200)
         self.assertEqual(
             self.client.get(reverse("dash-publishingstats-comprehensiveness-financials")).status_code, 200
@@ -88,6 +71,8 @@ def test_publishingstats_humanitarian(self):
 
     def test_exploringdata(self):
         """Test the exploring data pages"""
+        self.assertEqual(self.client.get(reverse("dash-headlines-files")).status_code, 200)
+        self.assertEqual(self.client.get(reverse("dash-headlines-activities")).status_code, 200)
         self.assertEqual(self.client.get(reverse("dash-exploringdata-booleans")).status_code, 200)
         self.assertEqual(self.client.get(reverse("dash-exploringdata-codelists")).status_code, 200)
         self.assertEqual(
@@ -127,10 +112,18 @@ def test_exploringdata(self):
             404,
         )
         self.assertEqual(self.client.get(reverse("dash-exploringdata-dates")).status_code, 200)
+        self.assertEqual(self.client.get(reverse("dash-versions")).status_code, 200)
+        self.assertEqual(self.client.get(reverse("dash-organisation")).status_code, 200)
+        self.assertEqual(self.client.get(reverse("dash-licenses")).status_code, 200)
+        self.assertEqual(self.client.get(reverse("dash-licenses-detail", args=("cc-by",))).status_code, 200)
+        self.assertEqual(
+            self.client.get(reverse("dash-licenses-detail", args=("not-a-valid-license",))).status_code,
+            404,
+        )
         self.assertEqual(self.client.get(reverse("dash-exploringdata-elements")).status_code, 200)
         self.assertEqual(
             self.client.get(
-                reverse("dash-exploringdata-elements-detail", args=("iati-activity_activity-date_narrative",))
+                reverse("dash-exploringdata-elements-detail", args=("iati-activity_activity-date_@iso-date",))
             ).status_code,
             200,
         )
@@ -172,7 +165,7 @@ def test_headlines_and_misc(self):
         self._url_and_view_helper(
             {
                 "index": "dash-index",
-                "headlines": "dash-headlines",
+                "headlines": "dash-index",
                 "files": "dash-headlines-files",
                 "activities": "dash-headlines-activities",
                 "publishers": "dash-headlines-publishers",
@@ -181,23 +174,23 @@ def test_headlines_and_misc(self):
             }
         )
 
-    def test_dataquality(self):
+    def test_errors(self):
         """Test data quality pages redirect to their new locations"""
 
         # This is not particularly DRY as a similar dictionary is created in views.py
         # but I think this is minor as that may disappear from views.py in a future
         # refactor of what goes into the context.
         self._url_and_view_helper(
             {
-                "data_quality": "dash-dataquality",
-                "download": "dash-dataquality-download",
-                "xml": "dash-dataquality-xml",
-                "validation": "dash-dataquality-validation",
-                "versions": "dash-dataquality-versions",
-                "organisation": "dash-dataquality-organisation",
-                "identifiers": "dash-dataquality-identifiers",
-                "reporting_orgs": "dash-dataquality-reportingorgs",
-                "licenses": "dash-dataquality-licenses",
+                "data_quality": "dash-errors",
+                "download": "dash-errors-download",
+                "xml": "dash-errors-xml",
+                "validation": "dash-errors-validation",
+                "versions": "dash-versions",
+                "organisation": "dash-organisation",
+                "identifiers": "dash-identifiers",
+                "reporting_orgs": "dash-errors-reportingorgs",
+                "licenses": "dash-licenses",
             }
         )
 
@@ -210,10 +203,10 @@ def test_publishingstats(self):
         self._url_and_view_helper(
             {
                 "publishing_stats": "dash-publishingstats",
-                "timeliness": "dash-publishingstats-timeliness",
+                "timeliness": "dash-publishingstats-timeliness-frequency",
                 "timeliness_timelag": "dash-publishingstats-timeliness-timelag",
                 "forwardlooking": "dash-publishingstats-forwardlooking",
-                "comprehensiveness": "dash-publishingstats-comprehensiveness",
+                "comprehensiveness": "dash-publishingstats-comprehensiveness-summary",
                 "comprehensiveness_core": "dash-publishingstats-comprehensiveness-core",
                 "comprehensiveness_financials": "dash-publishingstats-comprehensiveness-financials",
                 "comprehensiveness_valueadded": "dash-publishingstats-comprehensiveness-valueadded",
@@ -244,16 +237,14 @@ def test_slug_page_redirects(self):
         """Test pages with slugs redirect to the section page"""
 
         self.assertRedirects(
-            self.client.get(r"/publisher/undp.html"), reverse("dash-headlines-publishers"), status_code=301
-        )
-        self.assertRedirects(
-            self.client.get(r"/license/cc-by.html"), reverse("dash-dataquality-licenses"), status_code=301
+            self.client.get(r"/publisher/zsl.html"), reverse("dash-headlines-publishers"), status_code=301
         )
+        self.assertRedirects(self.client.get(r"/license/cc-by.html"), reverse("dash-licenses"), status_code=301)
         self.assertRedirects(
             self.client.get(r"/codelist/2/[email protected]"), reverse("dash-exploringdata-codelists"), status_code=301
         )
         self.assertRedirects(
-            self.client.get(r"/element/iati-activity_activity-date_narrative.html"),
+            self.client.get(r"/element/iati-activity_activity-date_@iso-date.html"),
             reverse("dash-exploringdata-elements"),
             status_code=301,
         )

diff --git a/iati_dashboard/ui/urls.py b/iati_dashboard/ui/urls.py
@@ -130,7 +130,8 @@
         ),
         # Redirects to support any users with bookmarks to pages on the old Dashboard.
         path("index.html", RedirectView.as_view(pattern_name="dash-index", permanent=True)),
-        path("headlines.html", RedirectView.as_view(pattern_name="dash-headlines", permanent=True)),
+        # We've remove the headlines page, so redirect to the index
+        path("headlines.html", RedirectView.as_view(pattern_name="dash-index", permanent=True)),
         path("data_quality.html", RedirectView.as_view(pattern_name="dash-errors", permanent=True)),
         path("exploring_data.html", RedirectView.as_view(pattern_name="dash-exploringdata-elements", permanent=True)),
         path("publishers.html", RedirectView.as_view(pattern_name="dash-headlines-publishers", permanent=True)),

diff --git a/iati_dashboard/ui/views.py b/iati_dashboard/ui/views.py
@@ -280,6 +280,9 @@ def headlines_files(request):
 
 
 def headlines_publisher_detail(request, publisher=None):
+    if publisher not in publisher_name:
+        raise Http404("Publisher does not exist")
+
     template = loader.get_template("publisher.html")
 
     context = _make_context("publishers")

diff --git a/mock_stats_for_unit_tests.sh b/mock_stats_for_unit_tests.sh
diff --git a/partial_stats_for_unit_tests.sh b/partial_stats_for_unit_tests.sh
@@ -0,0 +1,23 @@
+set -eux
+
+publisher_short_name=zsl
+
+if [ -d stats-calculated ]; then
+    echo "stats-calculated directory already exists, exiting."
+    exit
+fi
+wget -m --no-parent https://dev.merged.dashboard.iatistandard.org/stats/current/aggregated/
+wget -m --no-parent https://dev.merged.dashboard.iatistandard.org/stats/current/aggregated-publisher/$publisher_short_name/
+wget -m --no-parent https://dev.merged.dashboard.iatistandard.org/stats/current/aggregated-file/$publisher_short_name/
+wget -m --no-parent https://dev.merged.dashboard.iatistandard.org/stats/current/inverted-publisher/
+wget -m --no-parent https://dev.merged.dashboard.iatistandard.org/stats/current/inverted-file/
+wget -m --no-parent https://dev.merged.dashboard.iatistandard.org/stats/current/inverted-file-publisher/$publisher_short_name/
+wget -m --no-parent https://dev.merged.dashboard.iatistandard.org/stats/gitaggregate-publisher-dated/$publisher_short_name/
+mv dev.merged.dashboard.iatistandard.org/stats stats-calculated
+for file in ckan.json licenses.json gitdate.json; do
+    curl --compressed https://dev.merged.dashboard.iatistandard.org/stats/$file > stats-calculated/$file
+done
+
+cat stats-calculated/current/inverted-publisher/activities.json  | jq "{$publisher_short_name: .$publisher_short_name}" > activities.json
+mv activities.json stats-calculated/current/inverted-publisher/activities.json
+rm stats-calculated/current/aggregated-publisher/$publisher_short_name/index.html*