diff --git a/.talismanrc b/.talismanrc index d0f9a0dea..1f9ef3491 100644 --- a/.talismanrc +++ b/.talismanrc @@ -79,6 +79,8 @@ hub/workflows/e2e.yml checksum: df71affe33700cb0a36ee0b4824ef60e7d8e5a08f3d4d16ca53c31e9085b45ae - filename: queue/README.md checksum: fe2ac5277a42e6bf8dfc1e0fe53cc3308bba9019d9091f4b8c291e4a2e4a4eb8 +- filename: queue/src/.htpasswd + checksum: 7670652d46bed42247610009a41eadea1d18ff839db9fa831288cb6ab19792e1 - filename: queue/src/config.ts checksum: 8ca4d9a0b6118457fb7dcea24e43645eb63bea92d9e0f1d0c4ae763ea55491a5 - filename: queue/src/events/index.ts diff --git a/queue/package.json b/queue/package.json index 16eb16e91..9407552cf 100644 --- a/queue/package.json +++ b/queue/package.json @@ -14,6 +14,9 @@ "@aws-sdk/client-s3": "^3.577.0", "@aws-sdk/lib-storage": "^3.578.0", "@aws-sdk/s3-request-presigner": "^3.577.0", + "@bull-board/api": "5.20.2", + "@bull-board/express": "5.20.2", + "@bull-board/ui": "5.20.2", "@godaddy/terminus": "^4.12.1", "@sentry/integrations": "^7.116.0", "@sentry/node": "^7.116.0", @@ -29,6 +32,7 @@ "dotenv": "^16.4.5", "exceljs": "^4.4.0", "express": "^4.19.2", + "http-auth": "^4.2.0", "parse-redis-url-simple": "^1.0.2" }, "devDependencies": { @@ -38,6 +42,7 @@ "@types/async": "^3.2.24", "@types/convict": "^6.1.6", "@types/express": "^4.17.21", + "@types/http-auth": "^4", "@types/jest": "^29.5.12", "@types/node": "^20.12.12", "@types/supertest": "^6.0.2", diff --git a/queue/src/basic-auth.ts b/queue/src/basic-auth.ts new file mode 100644 index 000000000..6c176b1e9 --- /dev/null +++ b/queue/src/basic-auth.ts @@ -0,0 +1,8 @@ +import auth from 'http-auth'; +import path from 'node:path'; + +export const createBasicAuth = (): ReturnType => + auth.basic({ + file: path.join(__dirname, '.htpasswd'), + skipUser: true + }); diff --git a/queue/src/dashboard.ts b/queue/src/dashboard.ts new file mode 100644 index 000000000..a3af1b235 --- /dev/null +++ b/queue/src/dashboard.ts @@ -0,0 +1,28 @@ +import { ExpressAdapter } from '@bull-board/express'; +import { createBullBoard } from '@bull-board/api'; +import { BullMQAdapter } from '@bull-board/api/bullMQAdapter'; + +import { Queue } from 'bullmq'; +import { parseRedisUrl } from 'parse-redis-url-simple'; + +import { JOBS } from './jobs'; +import config from './config'; + +const expressAdapter = new ExpressAdapter(); +expressAdapter.setBasePath('/queues'); + +const [redis] = parseRedisUrl(config.redis.url); + +const queues = JOBS.map( + (job) => + new Queue(job, { + connection: redis + }) +).map((queue) => new BullMQAdapter(queue)); + +createBullBoard({ + queues, + serverAdapter: expressAdapter +}); + +export const createDashboard = () => expressAdapter.getRouter(); diff --git a/queue/src/server.ts b/queue/src/server.ts index 0b446705d..48f67d20e 100644 --- a/queue/src/server.ts +++ b/queue/src/server.ts @@ -5,10 +5,12 @@ import { healthcheck, postgresCheck, redisCheck, - s3Check, + s3Check } from '@zerologementvacant/healthcheck'; import config from './config'; import { createLogger } from './logger'; +import { createDashboard } from './dashboard'; +import { createBasicAuth } from './basic-auth'; function createServer() { const app = express(); @@ -22,12 +24,14 @@ function createServer() { checks: [ redisCheck(config.redis.url), postgresCheck(config.db.url), - s3Check(config.s3), + s3Check(config.s3) ], - logger, - }), + logger + }) ); + app.use('/queues', createBasicAuth().check(createDashboard())); + async function start(): Promise { const listen = util.promisify((port: number, cb: () => void) => { return app.listen(port, cb); @@ -39,7 +43,7 @@ function createServer() { return { app, - start, + start }; } diff --git a/yarn.lock b/yarn.lock index 3c5256d32..c2a2353ce 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2392,6 +2392,38 @@ __metadata: languageName: node linkType: hard +"@bull-board/api@npm:5.20.2": + version: 5.20.2 + resolution: "@bull-board/api@npm:5.20.2" + dependencies: + redis-info: "npm:^3.0.8" + peerDependencies: + "@bull-board/ui": 5.20.2 + checksum: 10c0/7bd8be3f776ac7d3adf47ba4a4b19aeb3eb32ab951f9e8212f793b3f2d3d0498c521fdf4a538851f596402c3b399ea982c2b8e6bcb01262952e9c976d5f13a96 + languageName: node + linkType: hard + +"@bull-board/express@npm:5.20.2": + version: 5.20.2 + resolution: "@bull-board/express@npm:5.20.2" + dependencies: + "@bull-board/api": "npm:5.20.2" + "@bull-board/ui": "npm:5.20.2" + ejs: "npm:^3.1.10" + express: "npm:^4.19.2" + checksum: 10c0/3e797f2f17ed091a58e7ed9d3d2cdb7d9fd0a9a55926535cf13eabb7b907d612301406e87e333145a3161d86882e3afbeb243c467ac20f3a738f44d8249d7924 + languageName: node + linkType: hard + +"@bull-board/ui@npm:5.20.2": + version: 5.20.2 + resolution: "@bull-board/ui@npm:5.20.2" + dependencies: + "@bull-board/api": "npm:5.20.2" + checksum: 10c0/9d1ec87d3de306737b5dff66bbfd8920e0d20fba0985466905ae4f3c30ed714de95cbb4757119846ba6cd46edd3147968163ea936805e855116aaded2ad98aa3 + languageName: node + linkType: hard + "@bundled-es-modules/cookie@npm:^2.0.0": version: 2.0.0 resolution: "@bundled-es-modules/cookie@npm:2.0.0" @@ -7923,6 +7955,15 @@ __metadata: languageName: node linkType: hard +"@types/http-auth@npm:^4": + version: 4.1.4 + resolution: "@types/http-auth@npm:4.1.4" + dependencies: + "@types/node": "npm:*" + checksum: 10c0/24a9497ef835c87b3df2d74fb6fd28960ce2921c944a20dbc5ba93e46561626d7381c8272659af930925f5149fe95f7515f892f5a9966ff89c91ffd6bdc8db02 + languageName: node + linkType: hard + "@types/http-errors@npm:*": version: 2.0.4 resolution: "@types/http-errors@npm:2.0.4" @@ -9151,6 +9192,9 @@ __metadata: "@aws-sdk/client-s3": "npm:^3.577.0" "@aws-sdk/lib-storage": "npm:^3.578.0" "@aws-sdk/s3-request-presigner": "npm:^3.577.0" + "@bull-board/api": "npm:5.20.2" + "@bull-board/express": "npm:5.20.2" + "@bull-board/ui": "npm:5.20.2" "@faker-js/faker": "npm:^8.4.1" "@godaddy/terminus": "npm:^4.12.1" "@sentry/integrations": "npm:^7.116.0" @@ -9160,6 +9204,7 @@ __metadata: "@types/async": "npm:^3.2.24" "@types/convict": "npm:^6.1.6" "@types/express": "npm:^4.17.21" + "@types/http-auth": "npm:^4" "@types/jest": "npm:^29.5.12" "@types/node": "npm:^20.12.12" "@types/supertest": "npm:^6.0.2" @@ -9175,6 +9220,7 @@ __metadata: dotenv: "npm:^16.4.5" exceljs: "npm:^4.4.0" express: "npm:^4.19.2" + http-auth: "npm:^4.2.0" jest: "npm:^29.7.0" jest-extended: "npm:^4.0.2" nodemon: "npm:^3.1.0" @@ -9674,6 +9720,22 @@ __metadata: languageName: node linkType: hard +"apache-crypt@npm:^1.1.2": + version: 1.2.6 + resolution: "apache-crypt@npm:1.2.6" + dependencies: + unix-crypt-td-js: "npm:^1.1.4" + checksum: 10c0/00ce45e671f256f3bbdcd47da57b9d9007af3e70102316304bfb1fb4f28610ea9b733e616a90079ee7c2bf1adebdda3fd99cafe2ac668ad55b0323c79df64f67 + languageName: node + linkType: hard + +"apache-md5@npm:^1.0.6": + version: 1.1.8 + resolution: "apache-md5@npm:1.1.8" + checksum: 10c0/423aa1baddcedc42e2fdf52efcf7fae2e7de9535e6ca7dd4a049f49fb5ec9b6a4469f327e02268088ed3dacdbec6f1ea4132941e2d75899c4e412421e6ffcbfc + languageName: node + linkType: hard + "append-field@npm:^1.0.0": version: 1.0.0 resolution: "append-field@npm:1.0.0" @@ -13096,7 +13158,7 @@ __metadata: languageName: node linkType: hard -"ejs@npm:^3.1.6": +"ejs@npm:^3.1.10, ejs@npm:^3.1.6": version: 3.1.10 resolution: "ejs@npm:3.1.10" dependencies: @@ -15694,6 +15756,18 @@ __metadata: languageName: node linkType: hard +"http-auth@npm:^4.2.0": + version: 4.2.0 + resolution: "http-auth@npm:4.2.0" + dependencies: + apache-crypt: "npm:^1.1.2" + apache-md5: "npm:^1.0.6" + bcryptjs: "npm:^2.4.3" + uuid: "npm:^8.3.2" + checksum: 10c0/9bd935ec6819e05c5b689db2f97fd81a9a1a0957074b0cbecc94251a47dc0b654e89ff8da9d091bd9ef16351e13a1fb8dda0a2e9352fb31bf137f3dbf5b3b28b + languageName: node + linkType: hard + "http-cache-semantics@npm:3.8.1": version: 3.8.1 resolution: "http-cache-semantics@npm:3.8.1" @@ -18715,7 +18789,7 @@ __metadata: languageName: node linkType: hard -"lodash@npm:^4.17.15, lodash@npm:^4.17.20, lodash@npm:^4.17.21, lodash@npm:^4.7.0": +"lodash@npm:^4.17.11, lodash@npm:^4.17.15, lodash@npm:^4.17.20, lodash@npm:^4.17.21, lodash@npm:^4.7.0": version: 4.17.21 resolution: "lodash@npm:4.17.21" checksum: 10c0/d8cbea072bb08655bb4c989da418994b073a608dffa608b09ac04b43a791b12aeae7cd7ad919aa4c925f33b48490b5cfe6c1f71d827956071dae2e7bb3a6b74c @@ -22971,6 +23045,15 @@ __metadata: languageName: node linkType: hard +"redis-info@npm:^3.0.8": + version: 3.1.0 + resolution: "redis-info@npm:3.1.0" + dependencies: + lodash: "npm:^4.17.11" + checksum: 10c0/ec0f31d97893c5828cec7166486d74198c92160c60073b6f2fe805cdf575a10ddcccc7641737d44b8f451355f0ab5b6c7b0d79e8fc24742b75dd625f91ffee38 + languageName: node + linkType: hard + "redis-parser@npm:^3.0.0": version: 3.0.0 resolution: "redis-parser@npm:3.0.0" @@ -25985,6 +26068,13 @@ __metadata: languageName: node linkType: hard +"unix-crypt-td-js@npm:^1.1.4": + version: 1.1.4 + resolution: "unix-crypt-td-js@npm:1.1.4" + checksum: 10c0/c4e3abd0d7ebcf39df7faff8be2cd137f477add743a2793c551682e04ec4e4f466e806a67e391d5a097229e4465b7cae4cb459990b9eb61dfe0b37d2388c6266 + languageName: node + linkType: hard + "unpipe@npm:1.0.0, unpipe@npm:~1.0.0": version: 1.0.0 resolution: "unpipe@npm:1.0.0"