Gov Paas to DBT PaaS migration (#652)

* TSS-1615: Pingdom (#644)

* TSS-1615: Pingdom

* format

---------

Co-authored-by: abarolo <[email protected]>

* TSS-1614: Xray + ASIM Logging (#645)

* TSS-1614: Xray

* Switch based on platform

* Handle all 3 loggers

---------

Co-authored-by: abarolo <[email protected]>

* Trigger pipeline

* Add .copilot directory for deployments

* Allow ELB health checks (#653)

* Allow ELB health checks

The ALLOWED_HOSTS variable needs to be dynamically
configured to allow health check requests from the ELB.
Use the DBT copilot python package helper function to do this.

* Alphabetically order imports

* Fix empty line

* Read new evnironment variable for CORS

ALLOWED_HOSTS isn't suitable as the automatic entry for the load balancer doesn't have a protocol.
The CORS_ALLOWED_ORIGINS values need to managed indepentently and contain https prefixes.

* Add debugging information for allowed hosts

* Debug Copilot environment variables

* More debugging for ALLOWED_HOSTS

* Bump Copilot python dependency

This includes a fix for the allowed_hosts helper function

* Remove debugging code

* Test Slack notifications with empty commit

* Remove ALLOWED_HOSTS debugging

* Temporarily disable error handling to see error connecting to Datahub

* Revert "Temporarily disable error handling to see error connecting to Datahub"

This reverts commit 820a970.

* Add APM to Sentry

* debug failed upload

* Fix syntax for printing responseCode

* Kick off another deployment

* Workaround for adding the node Docker layer

This is required to compile assets

* Update webpack React output filename

* Update `devtool` in Webpack configuration file

* Update Webpack configuration to use `source-map`

* Update error message in AttachmentForm

* Trigger pipeline

* Update requirements

* Merge master and sentry apm tracking

* chore: update Django configuration settings for Sentry

* Turn on static file compression

Enable static file compression and caching

* Tidy up

* PAAS Migration branch cleanup (#696)

* enhancing contrast colors (#693)

* Fix test and format

* Test fix

---------

Co-authored-by: Uka Osim <[email protected]>
Co-authored-by: abarolo <[email protected]>

* Use env.bool to check whether Elastic APM is enabled

---------

Co-authored-by: abarolo <[email protected]>
Co-authored-by: abarolo <[email protected]>
Co-authored-by: Emile Swarts <[email protected]>
Co-authored-by: Yusuf <[email protected]>
Co-authored-by: Gareth Pitt-Nash <[email protected]>
Co-authored-by: Uka Osim <[email protected]>

Author: 7 people

.copilot/config.yml

@@ -0,0 +1,4 @@
+repository: dmas/dmas-frontend
+builder:
+  name: paketobuildpacks/builder-jammy-full
+  version: 0.3.339

.copilot/image_build_run.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+# Exit early if something goes wrong
+set -e
+
+# Add commands below to run inside the container after all the other buildpacks have been applied

.copilot/phases/build.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+# Exit early if something goes wrong
+set -e
+
+# Add commands below to run as part of the build phase

.copilot/phases/install.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+# Exit early if something goes wrong
+set -e
+
+# Add commands below to run as part of the install phase

.copilot/phases/post_build.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+# Exit early if something goes wrong
+set -e
+
+# Add commands below to run as part of the post_build phase

.copilot/phases/pre_build.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+# Exit early if something goes wrong
+set -e
+
+# Add commands below to run as part of the pre_build phase

Procfile

@@ -1 +1,2 @@
-web: gunicorn config.wsgi:application --bind 0.0.0.0:$PORT --worker-class gevent --worker-connections 1000 --timeout 120 --log-file - --capture-output
+web: opentelemetry-instrument gunicorn config.wsgi:application --bind 0.0.0.0:$PORT --worker-class gevent --worker-connections 1000 --timeout 120 --log-file - --capture-output
+app: opentelemetry-instrument gunicorn config.wsgi:application --bind 0.0.0.0:$PORT --worker-class gevent --worker-connections 1000 --timeout 120 --log-file - --capture-output

config/settings/base.py

@@ -15,7 +15,12 @@
 from pathlib import Path
 
 import dj_database_url
+import requests
 import sentry_sdk
+from dbt_copilot_python.database import database_url_from_env
+from dbt_copilot_python.network import setup_allowed_hosts
+from dbt_copilot_python.utility import is_copilot
+from django_log_formatter_asim import ASIMFormatter
 from django_log_formatter_ecs import ECSFormatter
 from environ import Env
 from sentry_sdk.integrations.django import DjangoIntegration
@@ -45,9 +50,10 @@
 DEBUG = env("DEBUG")
 
 ALLOWED_HOSTS = env.list("ALLOWED_HOSTS", default=[])
-# Application definition
+ALLOWED_HOSTS = setup_allowed_hosts(ALLOWED_HOSTS)
 
-ELASTIC_APM_ENABLED = env("ELASTIC_APM_ENABLED", default=not DEBUG)
+# Application definition
+ELASTIC_APM_ENABLED = env.bool("ELASTIC_APM_ENABLED", default=not DEBUG)
 
 PRIORITISATION_STRATEGIC_ASSESSMENTS = env.bool(
     "PRIORITISATION_STRATEGIC_ASSESSMENTS", default=False
@@ -91,6 +97,7 @@
     "healthcheck",
     "reports",
     "users",
+    "pingdom",
 ]
 
 INSTALLED_APPS = BASE_APPS + DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS
@@ -148,11 +155,16 @@
 # Database
 # https://docs.djangoproject.com/en/3.0/ref/settings/#databases
 
-DATABASES = {
-    "default": dj_database_url.config(
-        env="DATABASE_URL", conn_max_age=0, conn_health_checks=True, default=""
-    )
-}
+if is_copilot():
+    DATABASES = {
+        "default": dj_database_url.config(
+            default=database_url_from_env("DATABASE_ENV_VAR_KEY"),
+            conn_max_age=0,
+            conn_health_checks=True,
+        )
+    }
+else:
+    DATABASES = {"default": env.db("DATABASE_URL")}
 
 
 # Password validation
@@ -196,12 +208,19 @@
 STATICFILES_DIRS = [
     str(ROOT_DIR / "core/frontend/dist/"),
 ]
-STATICFILES_STORAGE = "whitenoise.storage.CompressedStaticFilesStorage"
+
 STATICFILES_FINDERS = (
     "django.contrib.staticfiles.finders.FileSystemFinder",
     "django.contrib.staticfiles.finders.AppDirectoriesFinder",
 )
 
+STORAGES = {
+    "staticfiles": {
+        "BACKEND": env.str(
+            "STATICFILES_STORAGE", "whitenoise.storage.CompressedStaticFilesStorage"
+        ),
+    },
+}
 
 USER_DATA_CACHE_TIME = 3600
 METADATA_CACHE_TIME = "10600"
@@ -271,15 +290,19 @@
     "version": 1,
     "disable_existing_loggers": False,
     "formatters": {
-        "ecs_formatter": {
-            "()": ECSFormatter,
-        },
+        "asim_formatter": {"()": ASIMFormatter},
+        "ecs_formatter": {"()": ECSFormatter},
         "simple": {
             "format": "{asctime} {levelname} {message}",
             "style": "{",
         },
     },
     "handlers": {
+        "asim": {
+            "class": "logging.StreamHandler",
+            "stream": sys.stdout,  # noqa F405
+            "formatter": "asim_formatter",
+        },
         "ecs": {
             "class": "logging.StreamHandler",
             "stream": sys.stdout,  # noqa F405
@@ -292,47 +315,34 @@
         },
     },
     "root": {
-        "handlers": [
-            "ecs",
-            "stdout",
-        ],
+        "handlers": ["asim", "ecs", "stdout"],
         "level": os.getenv("ROOT_LOG_LEVEL", "INFO"),  # noqa F405
     },
     "loggers": {
-        "": {
-            "handlers": [
-                "ecs",
-                "stdout",
-            ],
-            "level": os.getenv("DJANGO_LOG_LEVEL", "INFO"),  # noqa F405
-            "propagate": False,
-        },
         "django": {
-            "handlers": [
-                "ecs",
-                "stdout",
-            ],
+            "handlers": ["asim", "ecs", "stdout"],
             "level": os.getenv("DJANGO_LOG_LEVEL", "INFO"),  # noqa F405
             "propagate": False,
         },
         "django.server": {
-            "handlers": [
-                "ecs",
-                "stdout",
-            ],
+            "handlers": ["asim", "ecs", "stdout"],
             "level": os.getenv("DJANGO_SERVER_LOG_LEVEL", "ERROR"),  # noqa F405
             "propagate": False,
         },
         "django.db.backends": {
-            "handlers": [
-                "ecs",
-                "stdout",
-            ],
+            "handlers": ["asim", "ecs", "stdout"],
             "level": os.getenv("DJANGO_DB_LOG_LEVEL", "ERROR"),  # noqa F405
             "propagate": False,
         },
     },
 }
+
+
+# Django Log Formatter ASIM settings
+if is_copilot():
+    DLFA_TRACE_HEADERS = ("X-B3-TraceId", "X-B3-SpanId")
+
+
 DLFE_APP_NAME = True
 DLFE_LOG_SENSITIVE_USER_DATA = True
 
@@ -343,9 +353,11 @@
 
 # Sentry
 SENTRY_BROWSER_TRACES_SAMPLE_RATE = env.float("SENTRY_BROWSER_TRACES_SAMPLE_RATE", 0.0)
-if not DEBUG:
+SENTRY_DNS = env("SENTRY_DSN", default=None)
+
+if SENTRY_DNS:
     sentry_sdk.init(
-        dsn=env("SENTRY_DSN"),
+        dsn=SENTRY_DNS,
         environment=env("SENTRY_ENVIRONMENT"),
         integrations=[
             DjangoIntegration(),

config/settings/hardening.py

@@ -18,4 +18,4 @@
 
 CSRF_COOKIE_HTTPONLY = True
 
-CORS_ALLOWED_ORIGINS = env.list("ALLOWED_HOSTS", default=[])
+CORS_ALLOWED_ORIGINS = env.list("CORS_ALLOWED_ORIGINS", default=[])

config/settings/test.py

@@ -8,8 +8,6 @@
 
 WHITENOISE_AUTOREFRESH = True
 
-STATICFILES_STORAGE = "django.contrib.staticfiles.storage.StaticFilesStorage"
-
 PASSWORD_HASHERS = [
     "django.contrib.auth.hashers.MD5PasswordHasher",
 ]

config/urls.py

@@ -19,6 +19,8 @@
 from django.contrib import admin
 from django.urls import include, path
 
+from pingdom.urls import urlpatterns as pingdom_urlpatterns
+
 urlpatterns = []
 
 if settings.DEBUG and settings.DJANGO_ENV == "local":
@@ -33,4 +35,4 @@
     path("", include("reports.urls", namespace="reports")),
     path("", include("core.urls", namespace="core")),
     path("", include("healthcheck.urls", namespace="healthcheck")),
-]
+] + pingdom_urlpatterns

config/wsgi.py

@@ -8,5 +8,7 @@
 """
 
 from django.core.wsgi import get_wsgi_application
+from opentelemetry.instrumentation.wsgi import OpenTelemetryMiddleware
 
 application = get_wsgi_application()
+application = OpenTelemetryMiddleware(application)

core/frontend/src/js/components/AttachmentForm.js

@@ -9,7 +9,7 @@ ma.components.AttachmentForm = (function (jessie) {
         fileUpload,
         attachments,
         submitButton,
-        multiDocument = true
+        multiDocument = true,
     ) {
         if (!fileUpload) {
             throw new Error("fileUpload is required");
@@ -44,7 +44,7 @@ ma.components.AttachmentForm = (function (jessie) {
                 this.fileUpload.setProgress(
                     "uploading file... " +
                         Math.floor((e.loaded / e.total) * 100) +
-                        "%"
+                        "%",
                 );
             }
         }
@@ -85,15 +85,16 @@ ma.components.AttachmentForm = (function (jessie) {
                 this.attachments.addItem(item, this.multiDocument);
             } else {
                 this.showError(
-                    "There was an issue uploading the document, try again"
+                    "There was an issue uploading the document, try again",
                 );
             }
         } else if (responseCode === 401) {
             this.showError(data.message);
         } else {
             var message =
-                data.message ||
-                "A system error has occured, so the file has not been uploaded. Try again.";
+                // data.message ||
+                //"A system error has occured, so the file has not been uploaded. Try again.;
+                responseCode;
             this.showError(message);
         }
     };
@@ -109,15 +110,15 @@ ma.components.AttachmentForm = (function (jessie) {
             xhr2.upload.addEventListener(
                 "progress",
                 bind(this.updateProgress, this),
-                false
+                false,
             );
         }
 
         xhr2.addEventListener("error", bind(this.transferFailed, this), false);
         xhr2.addEventListener(
             "abort",
             bind(this.transferCanceled, this),
-            false
+            false,
         );
         xhr2.addEventListener("load", bind(this.loaded, this), false);
 

package.json

@@ -9,7 +9,8 @@
     "build": "webpack --config webpack.config.js --mode production",
     "heroku-prebuild": "",
     "heroku-postbuild": "npm run build",
-    "lint": "eslint --fix --ext .js ./core/frontend/src/js"
+    "lint": "eslint --fix --ext .js ./core/frontend/src/js",
+    "start": "echo starting"
   },
   "dependencies": {
     "@babel/core": "^7.11.5",

pingdom/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class PingdomConfig(AppConfig):
+    name = "pingdom"
+    verbose_name = "Pingdom config"

pingdom/services.py

@@ -0,0 +1,17 @@
+from django.db import DatabaseError
+
+from healthcheck.models import HealthCheck
+
+
+class CheckDatabase:
+    name = "database"
+
+    def check(self):
+        try:
+            HealthCheck.objects.exists()
+            return True, ""
+        except DatabaseError as e:
+            return False, e
+
+
+services_to_check = (CheckDatabase,)

pingdom/urls.py

@@ -0,0 +1,7 @@
+from django.urls import path
+
+from pingdom.views import pingdom
+
+urlpatterns = [
+    path("pingdom/ping.xml", pingdom, name="pingdom"),
+]

pingdom/views.py

@@ -0,0 +1,27 @@
+from django.http import HttpResponse
+
+from pingdom.services import services_to_check
+
+PINGDOM_TEMPLATE = """<?xml version="1.0" encoding="UTF-8"?>
+<pingdom_http_custom_check>
+    <status>{status}</status>
+</pingdom_http_custom_check>\n"""
+
+COMMENT_TEMPLATE = "<!--{comment}-->\n"
+
+
+def pingdom(request):
+    print("processing")
+    checked = {}
+    for service in services_to_check:
+        checked[service.name] = service().check()
+
+    if all(item[0] for item in checked.values()):
+        return HttpResponse(
+            PINGDOM_TEMPLATE.format(status="OK"), content_type="text/xml"
+        )
+    else:
+        body = PINGDOM_TEMPLATE.format(status="FALSE")
+        for service_result in filter(lambda x: x[0] is False, checked.values()):
+            body += COMMENT_TEMPLATE.format(comment=service_result[1])
+        return HttpResponse(body, status=500, content_type="text/xml")