Add queue dashboard

.talismanrc

@@ -79,6 +79,8 @@ hub/workflows/e2e.yml
   checksum: df71affe33700cb0a36ee0b4824ef60e7d8e5a08f3d4d16ca53c31e9085b45ae
 - filename: queue/README.md
   checksum: fe2ac5277a42e6bf8dfc1e0fe53cc3308bba9019d9091f4b8c291e4a2e4a4eb8
+- filename: queue/src/.htpasswd
+  checksum: 7670652d46bed42247610009a41eadea1d18ff839db9fa831288cb6ab19792e1
 - filename: queue/src/config.ts
   checksum: 8ca4d9a0b6118457fb7dcea24e43645eb63bea92d9e0f1d0c4ae763ea55491a5
 - filename: queue/src/events/index.ts

queue/package.json

@@ -14,6 +14,9 @@
     "@aws-sdk/client-s3": "^3.577.0",
     "@aws-sdk/lib-storage": "^3.578.0",
     "@aws-sdk/s3-request-presigner": "^3.577.0",
+    "@bull-board/api": "5.20.2",
+    "@bull-board/express": "5.20.2",
+    "@bull-board/ui": "5.20.2",
     "@godaddy/terminus": "^4.12.1",
     "@sentry/integrations": "^7.116.0",
     "@sentry/node": "^7.116.0",
@@ -29,6 +32,7 @@
     "dotenv": "^16.4.5",
     "exceljs": "^4.4.0",
     "express": "^4.19.2",
+    "http-auth": "^4.2.0",
     "parse-redis-url-simple": "^1.0.2"
   },
   "devDependencies": {
@@ -38,6 +42,7 @@
     "@types/async": "^3.2.24",
     "@types/convict": "^6.1.6",
     "@types/express": "^4.17.21",
+    "@types/http-auth": "^4",
     "@types/jest": "^29.5.12",
     "@types/node": "^20.12.12",
     "@types/supertest": "^6.0.2",

queue/src/.htpasswd

@@ -0,0 +1 @@
+[email protected]:$apr1$6ts5byl3$575IQxAARDD6llSdjDpQM0

queue/src/basic-auth.ts

@@ -0,0 +1,8 @@
+import auth from 'http-auth';
+import path from 'node:path';
+
+export const createBasicAuth = () =>
+  auth.basic({
+    file: path.join(__dirname, '.htpasswd'),
+    skipUser: true
+  });

queue/src/dashboard.ts

@@ -0,0 +1,28 @@
+import { ExpressAdapter } from '@bull-board/express';
+import { createBullBoard } from '@bull-board/api';
+import { BullMQAdapter } from '@bull-board/api/bullMQAdapter';
+
+import { Queue } from 'bullmq';
+import { parseRedisUrl } from 'parse-redis-url-simple';
+
+import { JOBS } from './jobs';
+import config from './config';
+
+const expressAdapter = new ExpressAdapter();
+expressAdapter.setBasePath('/queues');
+
+const [redis] = parseRedisUrl(config.redis.url);
+
+const queues = JOBS.map(
+  (job) =>
+    new Queue(job, {
+      connection: redis
+    })
+).map((queue) => new BullMQAdapter(queue));
+
+createBullBoard({
+  queues,
+  serverAdapter: expressAdapter
+});
+
+export const createDashboard = () => expressAdapter.getRouter();

queue/src/server.ts

@@ -5,10 +5,12 @@ import {
   healthcheck,
   postgresCheck,
   redisCheck,
-  s3Check,
+  s3Check
 } from '@zerologementvacant/healthcheck';
 import config from './config';
 import { createLogger } from './logger';
+import { createDashboard } from './dashboard';
+import { createBasicAuth } from './basic-auth';
 
 function createServer() {
   const app = express();
@@ -22,12 +24,14 @@ function createServer() {
       checks: [
         redisCheck(config.redis.url),
         postgresCheck(config.db.url),
-        s3Check(config.s3),
+        s3Check(config.s3)
       ],
-      logger,
-    }),
+      logger
+    })
   );
 
+  app.use('/queues', createBasicAuth().check(createDashboard()));
+
   async function start(): Promise<void> {
     const listen = util.promisify((port: number, cb: () => void) => {
       return app.listen(port, cb);
@@ -39,7 +43,7 @@ function createServer() {
 
   return {
     app,
-    start,
+    start
   };
 }
 

yarn.lock

@@ -2392,6 +2392,38 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@bull-board/api@npm:5.20.2":
+  version: 5.20.2
+  resolution: "@bull-board/api@npm:5.20.2"
+  dependencies:
+    redis-info: "npm:^3.0.8"
+  peerDependencies:
+    "@bull-board/ui": 5.20.2
+  checksum: 10c0/7bd8be3f776ac7d3adf47ba4a4b19aeb3eb32ab951f9e8212f793b3f2d3d0498c521fdf4a538851f596402c3b399ea982c2b8e6bcb01262952e9c976d5f13a96
+  languageName: node
+  linkType: hard
+
+"@bull-board/express@npm:5.20.2":
+  version: 5.20.2
+  resolution: "@bull-board/express@npm:5.20.2"
+  dependencies:
+    "@bull-board/api": "npm:5.20.2"
+    "@bull-board/ui": "npm:5.20.2"
+    ejs: "npm:^3.1.10"
+    express: "npm:^4.19.2"
+  checksum: 10c0/3e797f2f17ed091a58e7ed9d3d2cdb7d9fd0a9a55926535cf13eabb7b907d612301406e87e333145a3161d86882e3afbeb243c467ac20f3a738f44d8249d7924
+  languageName: node
+  linkType: hard
+
+"@bull-board/ui@npm:5.20.2":
+  version: 5.20.2
+  resolution: "@bull-board/ui@npm:5.20.2"
+  dependencies:
+    "@bull-board/api": "npm:5.20.2"
+  checksum: 10c0/9d1ec87d3de306737b5dff66bbfd8920e0d20fba0985466905ae4f3c30ed714de95cbb4757119846ba6cd46edd3147968163ea936805e855116aaded2ad98aa3
+  languageName: node
+  linkType: hard
+
 "@bundled-es-modules/cookie@npm:^2.0.0":
   version: 2.0.0
   resolution: "@bundled-es-modules/cookie@npm:2.0.0"
@@ -7923,6 +7955,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/http-auth@npm:^4":
+  version: 4.1.4
+  resolution: "@types/http-auth@npm:4.1.4"
+  dependencies:
+    "@types/node": "npm:*"
+  checksum: 10c0/24a9497ef835c87b3df2d74fb6fd28960ce2921c944a20dbc5ba93e46561626d7381c8272659af930925f5149fe95f7515f892f5a9966ff89c91ffd6bdc8db02
+  languageName: node
+  linkType: hard
+
 "@types/http-errors@npm:*":
   version: 2.0.4
   resolution: "@types/http-errors@npm:2.0.4"
@@ -9151,6 +9192,9 @@ __metadata:
     "@aws-sdk/client-s3": "npm:^3.577.0"
     "@aws-sdk/lib-storage": "npm:^3.578.0"
     "@aws-sdk/s3-request-presigner": "npm:^3.577.0"
+    "@bull-board/api": "npm:5.20.2"
+    "@bull-board/express": "npm:5.20.2"
+    "@bull-board/ui": "npm:5.20.2"
     "@faker-js/faker": "npm:^8.4.1"
     "@godaddy/terminus": "npm:^4.12.1"
     "@sentry/integrations": "npm:^7.116.0"
@@ -9160,6 +9204,7 @@ __metadata:
     "@types/async": "npm:^3.2.24"
     "@types/convict": "npm:^6.1.6"
     "@types/express": "npm:^4.17.21"
+    "@types/http-auth": "npm:^4"
     "@types/jest": "npm:^29.5.12"
     "@types/node": "npm:^20.12.12"
     "@types/supertest": "npm:^6.0.2"
@@ -9175,6 +9220,7 @@ __metadata:
     dotenv: "npm:^16.4.5"
     exceljs: "npm:^4.4.0"
     express: "npm:^4.19.2"
+    http-auth: "npm:^4.2.0"
     jest: "npm:^29.7.0"
     jest-extended: "npm:^4.0.2"
     nodemon: "npm:^3.1.0"
@@ -9674,6 +9720,22 @@ __metadata:
   languageName: node
   linkType: hard
 
+"apache-crypt@npm:^1.1.2":
+  version: 1.2.6
+  resolution: "apache-crypt@npm:1.2.6"
+  dependencies:
+    unix-crypt-td-js: "npm:^1.1.4"
+  checksum: 10c0/00ce45e671f256f3bbdcd47da57b9d9007af3e70102316304bfb1fb4f28610ea9b733e616a90079ee7c2bf1adebdda3fd99cafe2ac668ad55b0323c79df64f67
+  languageName: node
+  linkType: hard
+
+"apache-md5@npm:^1.0.6":
+  version: 1.1.8
+  resolution: "apache-md5@npm:1.1.8"
+  checksum: 10c0/423aa1baddcedc42e2fdf52efcf7fae2e7de9535e6ca7dd4a049f49fb5ec9b6a4469f327e02268088ed3dacdbec6f1ea4132941e2d75899c4e412421e6ffcbfc
+  languageName: node
+  linkType: hard
+
 "append-field@npm:^1.0.0":
   version: 1.0.0
   resolution: "append-field@npm:1.0.0"
@@ -13096,7 +13158,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"ejs@npm:^3.1.6":
+"ejs@npm:^3.1.10, ejs@npm:^3.1.6":
   version: 3.1.10
   resolution: "ejs@npm:3.1.10"
   dependencies:
@@ -15694,6 +15756,18 @@ __metadata:
   languageName: node
   linkType: hard
 
+"http-auth@npm:^4.2.0":
+  version: 4.2.0
+  resolution: "http-auth@npm:4.2.0"
+  dependencies:
+    apache-crypt: "npm:^1.1.2"
+    apache-md5: "npm:^1.0.6"
+    bcryptjs: "npm:^2.4.3"
+    uuid: "npm:^8.3.2"
+  checksum: 10c0/9bd935ec6819e05c5b689db2f97fd81a9a1a0957074b0cbecc94251a47dc0b654e89ff8da9d091bd9ef16351e13a1fb8dda0a2e9352fb31bf137f3dbf5b3b28b
+  languageName: node
+  linkType: hard
+
 "http-cache-semantics@npm:3.8.1":
   version: 3.8.1
   resolution: "http-cache-semantics@npm:3.8.1"
@@ -18715,7 +18789,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"lodash@npm:^4.17.15, lodash@npm:^4.17.20, lodash@npm:^4.17.21, lodash@npm:^4.7.0":
+"lodash@npm:^4.17.11, lodash@npm:^4.17.15, lodash@npm:^4.17.20, lodash@npm:^4.17.21, lodash@npm:^4.7.0":
   version: 4.17.21
   resolution: "lodash@npm:4.17.21"
   checksum: 10c0/d8cbea072bb08655bb4c989da418994b073a608dffa608b09ac04b43a791b12aeae7cd7ad919aa4c925f33b48490b5cfe6c1f71d827956071dae2e7bb3a6b74c
@@ -22971,6 +23045,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"redis-info@npm:^3.0.8":
+  version: 3.1.0
+  resolution: "redis-info@npm:3.1.0"
+  dependencies:
+    lodash: "npm:^4.17.11"
+  checksum: 10c0/ec0f31d97893c5828cec7166486d74198c92160c60073b6f2fe805cdf575a10ddcccc7641737d44b8f451355f0ab5b6c7b0d79e8fc24742b75dd625f91ffee38
+  languageName: node
+  linkType: hard
+
 "redis-parser@npm:^3.0.0":
   version: 3.0.0
   resolution: "redis-parser@npm:3.0.0"
@@ -25985,6 +26068,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"unix-crypt-td-js@npm:^1.1.4":
+  version: 1.1.4
+  resolution: "unix-crypt-td-js@npm:1.1.4"
+  checksum: 10c0/c4e3abd0d7ebcf39df7faff8be2cd137f477add743a2793c551682e04ec4e4f466e806a67e391d5a097229e4465b7cae4cb459990b9eb61dfe0b37d2388c6266
+  languageName: node
+  linkType: hard
+
 "unpipe@npm:1.0.0, unpipe@npm:~1.0.0":
   version: 1.0.0
   resolution: "unpipe@npm:1.0.0"