Merge pull request #14 from DEFRA/test-coverage

adds tests, some bug fixes, adds test key override

Author: christopherjturner

src/config/index.js

@@ -377,6 +377,14 @@ const config = convict({
           default: 'key',
           env: 'PLATFORM_INTEGRATION_KEY'
         }
+      },
+      'platform-tenant-cko': {
+        integrationKey: {
+          doc: 'Integration key for digital service',
+          format: String,
+          default: 'key',
+          env: 'PLATFORM_TENANT_CKO_INTEGRATION_KEY'
+        }
       }
     },
     services: {

src/config/pagerduty-service-override.js

@@ -35,6 +35,9 @@ const serviceToPagerDutyServiceOverride = {
   },
   'cdp-waf': {
     teams: ['platform']
+  },
+  'cdp-canary': {
+    teams: ['platform-tenant-cko']
   }
 }
 

src/helpers/fetch/fetch-service.js

@@ -1,8 +1,9 @@
 import { config } from '~/src/config/index.js'
 
 /**
+ * Fetch Service info from portal backend.
  * @param {string} name
- * @returns {Promise<{teams: [{teamId: string}] }|null>}
+ * @returns {Promise<{teams: [{name: string}] }|null>}
  */
 async function fetchService(name) {
   const endpoint = config.get('portalBackendUrl') + `/services/${name}`

src/helpers/sqs/sqs-listener.js

@@ -70,7 +70,7 @@ const grafanaAlertListener = {
         server.logger.error(error, `SQS ${queueUrl}: ${error.message}`)
       }
       try {
-        await handleGrafanaPagerDutyAlert(message, server)
+        await handleGrafanaPagerDutyAlert(message)
         const receiptHandle = message.ReceiptHandle
         await deleteSqsMessage(server.sqs, queueUrl, receiptHandle)
       } catch (error) {

src/listeners/grafana/pagerduty/handle-grafana-pagerduty-alerts.js

@@ -2,31 +2,44 @@ import { config } from '~/src/config/index.js'
 import { serviceToPagerDutyServiceOverride } from '~/src/config/pagerduty-service-override.js'
 import { sendAlert } from '~/src/helpers/pagerduty/send-alert.js'
 import { fetchService } from '~/src/helpers/fetch/fetch-service.js'
-import { fetchTeam } from '~/src/helpers/fetch/fetch-team.js'
+
 import crypto from 'crypto'
+import { createLogger } from '~/src/helpers/logging/logger.js'
 
-const sendAlerts = config.get('pagerduty.sendAlerts')
+const logger = createLogger()
 
-function createDedupeKey(payload) {
-  const str = `${payload.summary}${payload.service}${payload.environment}${payload.startsAt}`
+/**
+ * Generates a pagerduty dedupe key based off the message's content.
+ * @param {{summary: string, service: string, environment: string, startsAt: string}|null} payload
+ * @returns {string}
+ */
+export function createDedupeKey(payload) {
+  const str = `${payload?.summary}${payload?.service}${payload?.environment}${payload?.startsAt}`
   return crypto.createHash('md5').update(str).digest('hex')
 }
 
 /**
- *
+ * Decides if an alert should be sent based off the environment it originated in.
  * @param {Alert} alert
+ * @param {string[]} environments
  * @returns {boolean}
  */
-function shouldSendAlert(alert) {
+export function shouldSendAlert(alert, environments) {
   const overrides =
     serviceToPagerDutyServiceOverride[alert.service]?.environments
 
-  const alertEnvironments = overrides || config.get('alertEnvironments')
+  const alertEnvironments = overrides || environments
   return alertEnvironments.includes(alert.environment)
 }
 
-async function getTeams(alert, logger) {
-  let teams = serviceToPagerDutyServiceOverride[alert.service]?.teams
+/**
+ * Gets the teams associated to an alert.
+ * Uses overrides first if set, otherwise uses owners of service from portal-backend.
+ * @param {Alert} alert
+ * @returns {Promise<string[]>}
+ */
+export async function getTeams(alert) {
+  const teams = serviceToPagerDutyServiceOverride[alert.service]?.teams
 
   if (teams) {
     return teams
@@ -40,62 +53,86 @@ async function getTeams(alert, logger) {
 
     return []
   }
-  teams = service.teams.map(async (team) => await fetchTeam(team.teamId))
-  return await Promise.all(teams)
+
+  return service.teams.map((team) => team.name)
 }
 
+/**
+ * Gets integration key for a team from config.js
+ * @param {string} team
+ * @returns {string|null}
+ */
 function findIntegrationKeyForTeam(team) {
-  return config.get(`pagerduty.teams.${team}.integrationKey`)
+  try {
+    return config.get(`pagerduty.teams.${team}.integrationKey`)
+  } catch {
+    return null
+  }
 }
 
-// If there is an integration key, it's assumed that we should send a PagerDuty alert
-function findIntegrationKeyForService(alert) {
+/**
+ * If there is an integration key, it's assumed that we should send a PagerDuty alert
+ * @param {Alert} alert
+ * @returns {string|null}
+ */
+export function findIntegrationKeyForService(alert) {
   const overrides =
     serviceToPagerDutyServiceOverride[alert.service]?.technicalService
 
   const service = overrides || alert.service
 
-  return config.get(`pagerduty.services.${service}.integrationKey`)
+  try {
+    return config.get(`pagerduty.services.${service}.integrationKey`)
+  } catch {
+    return null
+  }
 }
 
-async function handleGrafanaPagerDutyAlert(message, server) {
-  const logger = server.logger
+/**
+ *
+ * @param {{MessageId: string, Body: string}} message
+ * @returns {Promise<void>}
+ */
+export async function handleGrafanaPagerDutyAlert(message) {
   const payload = JSON.parse(message.Body)
 
-  if (!shouldSendAlert(payload)) {
+  if (!shouldSendAlert(payload, config.get('alertEnvironments'))) {
     return
   }
 
   if (!payload?.service) {
     logger.warn(
       `alert did not contain a service field:\n${JSON.stringify(payload)}`
     )
-    return []
+    return
   }
 
-  const teams = await getTeams(payload, logger)
+  const teams = await getTeams(payload)
 
-  let integrationKeys = teams
+  const integrationKeys = teams
     .map((team) => findIntegrationKeyForTeam(team))
     .filter((t) => t)
 
-  if (!integrationKeys) {
-    integrationKeys = [findIntegrationKeyForService(payload, server.logger)]
+  if (integrationKeys.length === 0) {
+    const key = findIntegrationKeyForService(payload)
+    if (key) {
+      integrationKeys.push(key)
+    }
   }
 
-  if (!integrationKeys.length) {
-    server.logger.debug(
+  if (integrationKeys.length === 0) {
+    logger.info(
       `No integration key found for ${payload.service}. Not sending alert - MessageId: ${message.MessageId}`
     )
     return
   }
 
-  server.logger.info(
+  logger.info(
     `Grafana alert ${payload.status} for ${payload.service} in ${payload.environment} - Alert: ${payload.alertName} MessageId: ${message.MessageId}`
   )
 
-  if (sendAlerts) {
-    server.logger.info('Sending PagerDuty alert')
+  if (config.get('pagerduty.sendAlerts')) {
+    logger.info('Sending PagerDuty alert')
 
     let eventAction
 
@@ -104,7 +141,7 @@ async function handleGrafanaPagerDutyAlert(message, server) {
     } else if (payload.status === 'resolved') {
       eventAction = 'resolve'
     } else {
-      server.logger.warn(
+      logger.warn(
         `Unexpected status ${payload.status} not sending alert:\n${JSON.stringify(payload)}`
       )
       return
@@ -115,8 +152,17 @@ async function handleGrafanaPagerDutyAlert(message, server) {
     const alerts = integrationKeys.map(async (integrationKey) => {
       await sendAlert(integrationKey, payload, teams, dedupeKey, eventAction)
     })
-    await Promise.all(alerts)
+    const result = await Promise.allSettled(alerts)
+    result
+      .filter((r) => r.status === 'rejected')
+      .forEach((r) => {
+        logger.error(`failed to send pager duty alert! ${r.reason}`)
+      })
+  } else {
+    logger.warn('NOT Sending PagerDuty alert (sendAlerts disabled in config)')
   }
 }
 
-export { handleGrafanaPagerDutyAlert }
+/**
+ * @import { Logger } from 'pino'
+ */