From efa4f3810973278c446052ed5519ebdc3cfdb882 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Fri, 25 Jun 2021 11:28:04 +0100 Subject: [PATCH 01/31] =?UTF-8?q?=F0=9F=91=B7=20Set=20up=20GitHub=20Action?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/test.yml | 38 ++++++++++++++++++++++++++++++++++++++ .travis.yml | 6 ------ package.json | 6 +++--- tag.sh | 26 ++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/test.yml delete mode 100644 .travis.yml create mode 100755 tag.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..9a96782 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,38 @@ +name: Test + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + build: + runs-on: ubuntu-18.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@v2 + with: + # Use PAT instead of default Github token, because the default + # token deliberately will not trigger another workflow run + token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + - uses: actions/setup-node@v2 + with: + node-version: '14.x' + registry-url: 'https://npm.pkg.github.com' + - name: Install + # Skip post-install to avoid malicious scripts stealing PAT + run: npm install --ignore-script + env: + # GITHUB_TOKEN can't access packages hosted in private repos, + # even within the same organisation + NODE_AUTH_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + - name: Post-install + run: npm rebuild && npm run prepare --if-present + - name: Test + run: npm test + - name: Tag + if: ${{ github.ref == 'refs/heads/main' }} + run: ./tag.sh diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 6342d3b..0000000 --- a/.travis.yml +++ /dev/null @@ -1,6 +0,0 @@ -language: node_js -node_js: - - "8" - - "10" - - "11" -services: mongodb diff --git a/package.json b/package.json index d2b77e9..9dc1f9f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { - "name": "mongodb-queue", - "version": "4.0.0", + "name": "@reedsy/mongodb-queue", + "version": "4.0.0-reedsy-1.0.0", "description": "Message queues which uses MongoDB.", "main": "mongodb-queue.js", "scripts": { @@ -15,7 +15,7 @@ "homepage": "https://github.com/chilts/mongodb-queue", "repository": { "type": "git", - "url": "git://github.com/chilts/mongodb-queue.git" + "url": "git://github.com/reedsy/mongodb-queue.git" }, "bugs": { "url": "http://github.com/chilts/mongodb-queue/issues", diff --git a/tag.sh b/tag.sh new file mode 100755 index 0000000..eee381f --- /dev/null +++ b/tag.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +VERSION=$(node -p "require('./package.json').version") + +git config --local user.email "github@reedsy.com" +git config --local user.name "GitHub Action" +git fetch --tags + +VERSION_COUNT=$(git tag --list $VERSION | wc -l) + +if [ $VERSION_COUNT -gt 0 ] +then + echo "Version $VERSION already deployed." + exit 0 +else + echo "Deploying version $VERSION" +fi + +echo '!/dist' >> .gitignore + +git checkout -b release-$VERSION +git add .gitignore +git add --all dist/ +git commit --message "Release version $VERSION" +git tag $VERSION +git push origin refs/tags/$VERSION From 8d42e8dc0477ae7197950cbd45d709436a5665fa Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Fri, 25 Jun 2021 11:30:14 +0100 Subject: [PATCH 02/31] =?UTF-8?q?=F0=9F=91=B7=20Add=20Mongo=20to=20GH=20Ac?= =?UTF-8?q?tions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9a96782..f794d77 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,6 +11,11 @@ on: jobs: build: runs-on: ubuntu-18.04 + services: + mongodb: + image: mongo:4.4 + ports: + - 27017:27017 timeout-minutes: 10 steps: - uses: actions/checkout@v2 From e97567c6f9796d0e84f89a9fcc913a28b3645307 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Fri, 25 Jun 2021 11:32:58 +0100 Subject: [PATCH 03/31] =?UTF-8?q?=F0=9F=91=B7=20Add=20Publish=20action?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/publish.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..861ca6a --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,30 @@ +name: Publish + +on: + push: + tags: + - '*' + +jobs: + build: + runs-on: ubuntu-18.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v2 + with: + node-version: '14.x' + registry-url: 'https://npm.pkg.github.com' + - name: Install + # Skip post-install to avoid malicious scripts stealing PAT + run: npm install --ignore-script + env: + # GITHUB_TOKEN can't access packages hosted in private repos, + # even within the same organisation + NODE_AUTH_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + - name: Post-install + run: npm rebuild && npm run prepare --if-present + - name: Publish + run: npm publish + env: + NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From b0d970002ab632542d0ba8d4a7612ff7c2483931 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Fri, 25 Jun 2021 16:02:13 +0100 Subject: [PATCH 04/31] =?UTF-8?q?=F0=9F=99=88=20Ignore=20`package-lock.jso?= =?UTF-8?q?n`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `package-lock.json` shouldn't be committed for libraries, since it's actively ignored by consuming apps. --- .gitignore | 1 + package-lock.json | 380 ---------------------------------------------- 2 files changed, 1 insertion(+), 380 deletions(-) delete mode 100644 package-lock.json diff --git a/.gitignore b/.gitignore index 04064b0..06ab5e5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ node_modules/* *.log *~ +package-lock.json diff --git a/package-lock.json b/package-lock.json deleted file mode 100644 index 7a5a278..0000000 --- a/package-lock.json +++ /dev/null @@ -1,380 +0,0 @@ -{ - "name": "mongodb-queue", - "version": "4.0.0", - "lockfileVersion": 1, - "requires": true, - "dependencies": { - "async": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/async/-/async-2.6.2.tgz", - "integrity": "sha512-H1qVYh1MYhEEFLsP97cVKqCGo7KfCyTt6uEWqsTBr9SO84oK9Uwbyd/yCW+6rKJLHksBNUVWZDAjfS+Ccx0Bbg==", - "dev": true, - "requires": { - "lodash": "^4.17.11" - } - }, - "balanced-match": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", - "dev": true - }, - "brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dev": true, - "requires": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "bson": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/bson/-/bson-1.1.0.tgz", - "integrity": "sha512-9Aeai9TacfNtWXOYarkFJRW2CWo+dRon+fuLZYJmvLV3+MiUp0bEI6IAZfXEIg7/Pl/7IWlLaDnhzTsD81etQA==", - "dev": true - }, - "concat-map": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", - "dev": true - }, - "deep-equal": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.0.1.tgz", - "integrity": "sha1-9dJgKStmDghO/0zbyfCK0yR0SLU=", - "dev": true - }, - "define-properties": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", - "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", - "dev": true, - "requires": { - "object-keys": "^1.0.12" - } - }, - "defined": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/defined/-/defined-1.0.0.tgz", - "integrity": "sha1-yY2bzvdWdBiOEQlpFRGZ45sfppM=", - "dev": true - }, - "es-abstract": { - "version": "1.13.0", - "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.13.0.tgz", - "integrity": "sha512-vDZfg/ykNxQVwup/8E1BZhVzFfBxs9NqMzGcvIJrqg5k2/5Za2bWo40dK2J1pgLngZ7c+Shh8lwYtLGyrwPutg==", - "dev": true, - "requires": { - "es-to-primitive": "^1.2.0", - "function-bind": "^1.1.1", - "has": "^1.0.3", - "is-callable": "^1.1.4", - "is-regex": "^1.0.4", - "object-keys": "^1.0.12" - } - }, - "es-to-primitive": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.0.tgz", - "integrity": "sha512-qZryBOJjV//LaxLTV6UC//WewneB3LcXOL9NP++ozKVXsIIIpm/2c13UDiD9Jp2eThsecw9m3jPqDwTyobcdbg==", - "dev": true, - "requires": { - "is-callable": "^1.1.4", - "is-date-object": "^1.0.1", - "is-symbol": "^1.0.2" - } - }, - "for-each": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", - "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==", - "dev": true, - "requires": { - "is-callable": "^1.1.3" - } - }, - "fs.realpath": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", - "dev": true - }, - "function-bind": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", - "dev": true - }, - "glob": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", - "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", - "dev": true, - "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - } - }, - "has": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", - "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "dev": true, - "requires": { - "function-bind": "^1.1.1" - } - }, - "has-symbols": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.0.tgz", - "integrity": "sha1-uhqPGvKg/DllD1yFA2dwQSIGO0Q=", - "dev": true - }, - "inflight": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", - "dev": true, - "requires": { - "once": "^1.3.0", - "wrappy": "1" - } - }, - "inherits": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=", - "dev": true - }, - "is-callable": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.4.tgz", - "integrity": "sha512-r5p9sxJjYnArLjObpjA4xu5EKI3CuKHkJXMhT7kwbpUyIFD1n5PMAsoPvWnvtZiNz7LjkYDRZhd7FlI0eMijEA==", - "dev": true - }, - "is-date-object": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.1.tgz", - "integrity": "sha1-mqIOtq7rv/d/vTPnTKAbM1gdOhY=", - "dev": true - }, - "is-regex": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.4.tgz", - "integrity": "sha1-VRdIm1RwkbCTDglWVM7SXul+lJE=", - "dev": true, - "requires": { - "has": "^1.0.1" - } - }, - "is-symbol": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.2.tgz", - "integrity": "sha512-HS8bZ9ox60yCJLH9snBpIwv9pYUAkcuLhSA1oero1UB5y9aiQpRA8y2ex945AOtCZL1lJDeIk3G5LthswI46Lw==", - "dev": true, - "requires": { - "has-symbols": "^1.0.0" - } - }, - "lodash": { - "version": "4.17.15", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", - "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", - "dev": true - }, - "memory-pager": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", - "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", - "dev": true, - "optional": true - }, - "minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", - "dev": true, - "requires": { - "brace-expansion": "^1.1.7" - } - }, - "minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", - "dev": true - }, - "mongodb": { - "version": "3.1.13", - "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.1.13.tgz", - "integrity": "sha512-sz2dhvBZQWf3LRNDhbd30KHVzdjZx9IKC0L+kSZ/gzYquCF5zPOgGqRz6sSCqYZtKP2ekB4nfLxhGtzGHnIKxA==", - "dev": true, - "requires": { - "mongodb-core": "3.1.11", - "safe-buffer": "^5.1.2" - } - }, - "mongodb-core": { - "version": "3.1.11", - "resolved": "https://registry.npmjs.org/mongodb-core/-/mongodb-core-3.1.11.tgz", - "integrity": "sha512-rD2US2s5qk/ckbiiGFHeu+yKYDXdJ1G87F6CG3YdaZpzdOm5zpoAZd/EKbPmFO6cQZ+XVXBXBJ660sSI0gc6qg==", - "dev": true, - "requires": { - "bson": "^1.1.0", - "require_optional": "^1.0.1", - "safe-buffer": "^5.1.2", - "saslprep": "^1.0.0" - } - }, - "object-inspect": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.6.0.tgz", - "integrity": "sha512-GJzfBZ6DgDAmnuaM3104jR4s1Myxr3Y3zfIyN4z3UdqN69oSRacNK8UhnobDdC+7J2AHCjGwxQubNJfE70SXXQ==", - "dev": true - }, - "object-keys": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.0.tgz", - "integrity": "sha512-6OO5X1+2tYkNyNEx6TsCxEqFfRWaqx6EtMiSbGrw8Ob8v9Ne+Hl8rBAgLBZn5wjEz3s/s6U1WXFUFOcxxAwUpg==", - "dev": true - }, - "once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", - "dev": true, - "requires": { - "wrappy": "1" - } - }, - "path-is-absolute": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", - "dev": true - }, - "path-parse": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", - "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==", - "dev": true - }, - "require_optional": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/require_optional/-/require_optional-1.0.1.tgz", - "integrity": "sha512-qhM/y57enGWHAe3v/NcwML6a3/vfESLe/sGM2dII+gEO0BpKRUkWZow/tyloNqJyN6kXSl3RyyM8Ll5D/sJP8g==", - "dev": true, - "requires": { - "resolve-from": "^2.0.0", - "semver": "^5.1.0" - } - }, - "resolve": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.10.0.tgz", - "integrity": "sha512-3sUr9aq5OfSg2S9pNtPA9hL1FVEAjvfOC4leW0SNf/mpnaakz2a9femSd6LqAww2RaFctwyf1lCqnTHuF1rxDg==", - "dev": true, - "requires": { - "path-parse": "^1.0.6" - } - }, - "resolve-from": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-2.0.0.tgz", - "integrity": "sha1-lICrIOlP+h2egKgEx+oUdhGWa1c=", - "dev": true - }, - "resumer": { - "version": "0.0.0", - "resolved": "https://registry.npmjs.org/resumer/-/resumer-0.0.0.tgz", - "integrity": "sha1-8ej0YeQGS6Oegq883CqMiT0HZ1k=", - "dev": true, - "requires": { - "through": "~2.3.4" - } - }, - "safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", - "dev": true - }, - "saslprep": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.2.tgz", - "integrity": "sha512-4cDsYuAjXssUSjxHKRe4DTZC0agDwsCqcMqtJAQPzC74nJ7LfAJflAtC1Zed5hMzEQKj82d3tuzqdGNRsLJ4Gw==", - "dev": true, - "optional": true, - "requires": { - "sparse-bitfield": "^3.0.3" - } - }, - "semver": { - "version": "5.6.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz", - "integrity": "sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg==", - "dev": true - }, - "sparse-bitfield": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", - "integrity": "sha1-/0rm5oZWBWuks+eSqzM004JzyhE=", - "dev": true, - "optional": true, - "requires": { - "memory-pager": "^1.0.2" - } - }, - "string.prototype.trim": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.1.2.tgz", - "integrity": "sha1-0E3iyJ4Tf019IG8Ia17S+ua+jOo=", - "dev": true, - "requires": { - "define-properties": "^1.1.2", - "es-abstract": "^1.5.0", - "function-bind": "^1.0.2" - } - }, - "tape": { - "version": "4.10.1", - "resolved": "https://registry.npmjs.org/tape/-/tape-4.10.1.tgz", - "integrity": "sha512-G0DywYV1jQeY3axeYnXUOt6ktnxS9OPJh97FGR3nrua8lhWi1zPflLxcAHavZ7Jf3qUfY7cxcVIVFa4mY2IY1w==", - "dev": true, - "requires": { - "deep-equal": "~1.0.1", - "defined": "~1.0.0", - "for-each": "~0.3.3", - "function-bind": "~1.1.1", - "glob": "~7.1.3", - "has": "~1.0.3", - "inherits": "~2.0.3", - "minimist": "~1.2.0", - "object-inspect": "~1.6.0", - "resolve": "~1.10.0", - "resumer": "~0.0.0", - "string.prototype.trim": "~1.1.2", - "through": "~2.3.8" - } - }, - "through": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=", - "dev": true - }, - "wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", - "dev": true - } - } -} From f415bea1f2cfde95edd8ab6620b8dd0e79169cdd Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Fri, 25 Jun 2021 16:01:15 +0100 Subject: [PATCH 05/31] Option to reset tries in `ping()` The `ping()` method can be useful for setting up recurring jobs if we deliberately avoid acking the job. For example: 1. Submit job 2. Pull job from queue 3. Process job 4. `ping()` 5. Go to Step 2 A real-world example of this might be notifying for recurring appointments, or setting up long-running, cross-process, periodic jobs. The main advantage this has over using `ack()` and `add()` is that it effectively requeues a job in a single, atomic commit. If we tried the above with `ack()` and `add()`: 1. Submit job 2. Pull job from queue 3. Process job 4. `ack()` 5. `add()` 6. Go to Step 2 In this version, the process could crash or quit between Steps 4 & 5, and our recurring job would be lost. We could also try inverting Steps 4 & 5, but then we get the opposite issue: if the process crashes or quits, then we might accidentally duplicate our recurring job. It also prevents us from setting up any unique indexes on our `payload`. Using `ping()` perfectly solves this problem: there's only ever one version of the job, and it's never dropped (because it's never acked). If the process crashes before we `ping()`, we'll retry it, as with any other normal job. The one issue with this approach is that `tries` will steadily increase, and - if you have `maxRetries` set up - the job will eventually be moved to the dead queue, which isn't what we want. This change adds an option to the `ping()` method: `resetTries`, which will reset `tries` to zero, so that the job is treated like a "new" job when it's pinged, and is only moved to the dead queue if it's genuinely retried. --- README.md | 11 ++++++++++ mongodb-queue.js | 5 +++++ test/ping.js | 55 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+) diff --git a/README.md b/README.md index 57a7206..f3c2be1 100644 --- a/README.md +++ b/README.md @@ -345,6 +345,17 @@ queue.get((err, msg) => { }) ``` +You can also reset the job tries, effectively creating an atomic ack + add for the +same job using `resetTries`: + +```js +queue.get((err, msg) => { + queue.ping(msg.ack, { resetTries: true }, (err, id) => { + // This message now has 0 tries + }) +}) +``` + ### .total() ### Returns the total number of messages that has ever been in the queue, including diff --git a/mongodb-queue.js b/mongodb-queue.js index a712f1c..20d3833 100644 --- a/mongodb-queue.js +++ b/mongodb-queue.js @@ -175,6 +175,11 @@ Queue.prototype.ping = function(ack, opts, callback) { visible : nowPlusSecs(visibility) } } + + if (opts.resetTries) { + update.$set.tries = 0 + } + self.col.findOneAndUpdate(query, update, { returnOriginal : false }, function(err, msg, blah) { if (err) return callback(err) if ( !msg.value ) { diff --git a/test/ping.js b/test/ping.js index 298786a..5c61593 100644 --- a/test/ping.js +++ b/test/ping.js @@ -174,6 +174,61 @@ test("ping: check visibility option overrides the queue visibility", function(t) ) }) + test("ping: reset tries", function(t) { + var queue = mongoDbQueue(db, 'ping', { visibility: 3 }) + var msg + + async.series( + [ + function(next) { + queue.add('Hello, World!', function(err, id) { + t.ok(!err, 'There is no error when adding a message.') + t.ok(id, 'There is an id returned when adding a message.') + next() + }) + }, + function(next) { + queue.get(function(err, thisMsg) { + msg = thisMsg + // message should reset in three seconds + t.ok(msg.id, 'Got a msg.id (sanity check)') + setTimeout(next, 2 * 1000) + }) + }, + function(next) { + queue.ping(msg.ack, { resetTries: true }, function(err, id) { + t.ok(!err, 'No error when pinging a message') + t.ok(id, 'Received an id when acking this message') + // wait until the msg has returned to the queue + setTimeout(next, 6 * 1000) + }) + }, + function(next) { + queue.get(function(err, msg) { + t.equal(msg.tries, 1, 'Tries were reset') + queue.ack(msg.ack, function(err) { + t.ok(!err, 'No error when acking the message') + next() + }) + }) + }, + function(next) { + queue.get(function(err, msg) { + // no more messages + t.ok(!err, 'No error when getting no messages') + t.ok(!msg, 'No msg received') + next() + }) + } + ], + function(err) { + if (err) t.fail(err) + t.pass('Finished test ok') + t.end() + } + ) + }) + test('client.close()', function(t) { t.pass('client.close()') client.close() From 182ca32ac3e7ff90caf0b478f612dc75a3301888 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 29 Jun 2021 14:26:05 +0100 Subject: [PATCH 06/31] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20`mongodb`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 9dc1f9f..de5a5e0 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "dependencies": {}, "devDependencies": { "async": "^2.6.2", - "mongodb": "^3.1.13", + "mongodb": "^3.6.9", "tape": "^4.10.1" }, "homepage": "https://github.com/chilts/mongodb-queue", From 804fa222e05ed0de1204ae2ff94570199c052db3 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 29 Jun 2021 14:40:15 +0100 Subject: [PATCH 07/31] =?UTF-8?q?=F0=9F=91=BD=20Add=20option=20to=20use=20?= =?UTF-8?q?`returnDocument`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `mongodb` Node.js driver deprecated use of `returnOriginal` in favour of `returnDocument` in [v3.6][1]. This non-breaking change allows consumers to opt in to using the newer `returnDocument` by setting an option on construction ```js var queue = mongoDbQueue(db, 'queue', { returnDocument : true }) ``` [1]: https://github.com/mongodb/node-mongodb-native/pull/2808 --- README.md | 12 ++++++++++++ mongodb-queue.js | 21 ++++++++++++++++++--- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f3c2be1..759b2e7 100644 --- a/README.md +++ b/README.md @@ -237,6 +237,18 @@ msg = { Notice that the payload from the `deadQueue` is exactly the same as the original message when it was on the original queue (except with the number of tries set to 5). +### returnDocument ### + +The `mongodb` Node.js driver [deprecated](https://github.com/mongodb/node-mongodb-native/pull/2808) +use of `returnOriginal` in favor of `returnDocument` when using `findOneAndUpdate()`. + +If you want to opt in to using the newer `returnDocument`, set the `returnDocument` option +to `true`: + +``` +var queue = mongoDbQueue(db, 'queue', { returnDocument : true }) +``` + ## Operations ## ### .add() ### diff --git a/mongodb-queue.js b/mongodb-queue.js index 20d3833..5a0f5d5 100644 --- a/mongodb-queue.js +++ b/mongodb-queue.js @@ -44,6 +44,7 @@ function Queue(db, name, opts) { this.col = db.collection(name) this.visibility = opts.visibility || 30 this.delay = opts.delay || 0 + this.returnDocument = opts.returnDocument if ( opts.deadQueue ) { this.deadQueue = opts.deadQueue @@ -120,8 +121,11 @@ Queue.prototype.get = function(opts, callback) { visible : nowPlusSecs(visibility), } } + var options = self._optionsWithNewDocument({ + sort: sort + }) - self.col.findOneAndUpdate(query, update, { sort: sort, returnOriginal : false }, function(err, result) { + self.col.findOneAndUpdate(query, update, options, function(err, result) { if (err) return callback(err) var msg = result.value if (!msg) return callback() @@ -175,12 +179,13 @@ Queue.prototype.ping = function(ack, opts, callback) { visible : nowPlusSecs(visibility) } } + var options = self._optionsWithNewDocument({}) if (opts.resetTries) { update.$set.tries = 0 } - self.col.findOneAndUpdate(query, update, { returnOriginal : false }, function(err, msg, blah) { + self.col.findOneAndUpdate(query, update, options, function(err, msg, blah) { if (err) return callback(err) if ( !msg.value ) { return callback(new Error("Queue.ping(): Unidentified ack : " + ack)) @@ -202,7 +207,8 @@ Queue.prototype.ack = function(ack, callback) { deleted : now(), } } - self.col.findOneAndUpdate(query, update, { returnOriginal : false }, function(err, msg, blah) { + var options = self._optionsWithNewDocument({}) + self.col.findOneAndUpdate(query, update, options, function(err, msg, blah) { if (err) return callback(err) if ( !msg.value ) { return callback(new Error("Queue.ack(): Unidentified ack : " + ack)) @@ -271,3 +277,12 @@ Queue.prototype.done = function(callback) { callback(null, count) }) } + +Queue.prototype._optionsWithNewDocument = function(query) { + if (this.returnDocument) { + query.returnDocument = 'after' + } else { + query.returnOriginal = false + } + return query +} From 341d8e20a517f8cd7de46dfbbea7e8d734297692 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 29 Jun 2021 16:00:42 +0100 Subject: [PATCH 08/31] =?UTF-8?q?=F0=9F=94=96=20v4.0.0-reedsy-1.1.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index de5a5e0..b0f3a4e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@reedsy/mongodb-queue", - "version": "4.0.0-reedsy-1.0.0", + "version": "4.0.0-reedsy-1.1.0", "description": "Message queues which uses MongoDB.", "main": "mongodb-queue.js", "scripts": { From 39ed0655568c456bb41a5129a19e0bd5552ef1fa Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Fri, 23 Jul 2021 09:58:50 +0100 Subject: [PATCH 09/31] =?UTF-8?q?=F0=9F=92=A5=20Require=20`mongodb`=20v4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change bumps the `mongodb` dependency to v4, and: - removes the `returnDocument` option: the deprecated version is no longer supported for v4, and we drop its support here - update `add()` to check `insertedIds`, since `ops` was removed in v4 --- mongodb-queue.js | 29 ++++++++++++----------------- package.json | 7 +++++-- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/mongodb-queue.js b/mongodb-queue.js index 5a0f5d5..61f3f82 100644 --- a/mongodb-queue.js +++ b/mongodb-queue.js @@ -44,7 +44,6 @@ function Queue(db, name, opts) { this.col = db.collection(name) this.visibility = opts.visibility || 30 this.delay = opts.delay || 0 - this.returnDocument = opts.returnDocument if ( opts.deadQueue ) { this.deadQueue = opts.deadQueue @@ -95,7 +94,7 @@ Queue.prototype.add = function(payload, opts, callback) { self.col.insertMany(msgs, function(err, results) { if (err) return callback(err) if (payload instanceof Array) return callback(null, '' + results.insertedIds) - callback(null, '' + results.ops[0]._id) + callback(null, '' + results.insertedIds[0]) }) } @@ -121,9 +120,10 @@ Queue.prototype.get = function(opts, callback) { visible : nowPlusSecs(visibility), } } - var options = self._optionsWithNewDocument({ - sort: sort - }) + var options = { + sort: sort, + returnDocument: 'after' + } self.col.findOneAndUpdate(query, update, options, function(err, result) { if (err) return callback(err) @@ -179,7 +179,9 @@ Queue.prototype.ping = function(ack, opts, callback) { visible : nowPlusSecs(visibility) } } - var options = self._optionsWithNewDocument({}) + var options = { + returnDocument: 'after' + } if (opts.resetTries) { update.$set.tries = 0 @@ -207,8 +209,10 @@ Queue.prototype.ack = function(ack, callback) { deleted : now(), } } - var options = self._optionsWithNewDocument({}) - self.col.findOneAndUpdate(query, update, options, function(err, msg, blah) { + var options = { + returnDocument: 'after' + } + self.col.findOneAndUpdate(query, update, options, function(err, msg) { if (err) return callback(err) if ( !msg.value ) { return callback(new Error("Queue.ack(): Unidentified ack : " + ack)) @@ -277,12 +281,3 @@ Queue.prototype.done = function(callback) { callback(null, count) }) } - -Queue.prototype._optionsWithNewDocument = function(query) { - if (this.returnDocument) { - query.returnDocument = 'after' - } else { - query.returnOriginal = false - } - return query -} diff --git a/package.json b/package.json index b0f3a4e..9c4e884 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@reedsy/mongodb-queue", - "version": "4.0.0-reedsy-1.1.0", + "version": "4.0.0-reedsy-2.0.0", "description": "Message queues which uses MongoDB.", "main": "mongodb-queue.js", "scripts": { @@ -9,9 +9,12 @@ "dependencies": {}, "devDependencies": { "async": "^2.6.2", - "mongodb": "^3.6.9", + "mongodb": "^4.0.0", "tape": "^4.10.1" }, + "peerDependencies": { + "mongodb": "^4.0.0" + }, "homepage": "https://github.com/chilts/mongodb-queue", "repository": { "type": "git", From 0b162cbed28ae53c2759d2b2d7c4c96924cb0249 Mon Sep 17 00:00:00 2001 From: Dawid Kisielewski Date: Thu, 24 Nov 2022 10:56:55 +0100 Subject: [PATCH 10/31] =?UTF-8?q?=E2=9A=A1=20Add=20deleted=20index?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mongodb-queue.js | 5 ++++- package.json | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/mongodb-queue.js b/mongodb-queue.js index 61f3f82..6e5f1b0 100644 --- a/mongodb-queue.js +++ b/mongodb-queue.js @@ -58,7 +58,10 @@ Queue.prototype.createIndexes = function(callback) { if (err) return callback(err) self.col.createIndex({ ack : 1 }, { unique : true, sparse : true }, function(err) { if (err) return callback(err) - callback(null, indexname) + self.col.createIndex({ deleted : 1 }, { sparse : true }, function(err) { + if (err) return callback(err) + callback(null, indexname) + }) }) }) } diff --git a/package.json b/package.json index 9c4e884..6fe3e12 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@reedsy/mongodb-queue", - "version": "4.0.0-reedsy-2.0.0", + "version": "4.0.0-reedsy-2.0.1", "description": "Message queues which uses MongoDB.", "main": "mongodb-queue.js", "scripts": { From f5c4758bcb78f9fdb1e72ca1d939e94f020e70f1 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Wed, 1 Mar 2023 15:48:27 +0000 Subject: [PATCH 11/31] =?UTF-8?q?=F0=9F=92=A5=20Promisify=20and=20add=20`m?= =?UTF-8?q?ongodb@5`=20support?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [`mongodb@5`][1] drops support for callbacks, which breaks this library, which is all written with callbacks. This is a **BREAKING** change which drops callback support from this library as well, and fully embraces promises through `async`/`await` syntax in both the library code and test code. This allows us to support both `mongodb@4` and `mongodb@5`. [1]: https://github.com/mongodb/node-mongodb-native/releases/tag/v5.0.0 --- README.md | 22 +-- mongodb-queue.js | 352 ++++++++++++++++++--------------------------- package.json | 7 +- test/_timeout.js | 7 + test/clean.js | 193 +++++-------------------- test/dead-queue.js | 230 +++++++++-------------------- test/default.js | 143 +++++++----------- test/delay.js | 147 +++++++------------ test/indexes.js | 23 ++- test/many.js | 104 +++++--------- test/multi.js | 86 ++++------- test/ping.js | 330 ++++++++++++++---------------------------- test/setup.js | 25 +--- test/stats.js | 235 +++++++----------------------- test/visibility.js | 243 +++++++++++-------------------- 15 files changed, 679 insertions(+), 1468 deletions(-) create mode 100644 test/_timeout.js diff --git a/README.md b/README.md index 759b2e7..5684d76 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ const client = new mongodb.MongoClient(url, { useNewUrlParser: true }) client.connect(err => { const db = client.db('test') - const queue = mongoDbQueue(db, 'my-queue') + const queue = new MongoDbQueue(db, 'my-queue') // ... @@ -102,9 +102,9 @@ passed in: var mongoDbQueue = require('mongodb-queue') // an instance of a queue -var queue1 = mongoDbQueue(db, 'a-queue') +var queue1 = new MongoDbQueue(db, 'a-queue') // another queue which uses the same collection as above -var queue2 = mongoDbQueue(db, 'a-queue') +var queue2 = new MongoDbQueue(db, 'a-queue') ``` Using `queue1` and `queue2` here won't interfere with each other and will play along nicely, but that's not @@ -116,7 +116,7 @@ it's not something you should do. To pass in options for the queue: ``` -var resizeQueue = mongoDbQueue(db, 'resize-queue', { visibility : 30, delay : 15 }) +var resizeQueue = new MongoDbQueue(db, 'resize-queue', { visibility : 30, delay : 15 }) ``` This example shows a queue with a message visibility of 30s and a delay to each message of 15s. @@ -131,8 +131,8 @@ Each queue you create will be it's own collection. e.g. ``` -var resizeImageQueue = mongoDbQueue(db, 'resize-image-queue') -var notifyOwnerQueue = mongoDbQueue(db, 'notify-owner-queue') +var resizeImageQueue = new MongoDbQueue(db, 'resize-image-queue') +var notifyOwnerQueue = new MongoDbQueue(db, 'notify-owner-queue') ``` This will create two collections in MongoDB called `resize-image-queue` and `notify-owner-queue`. @@ -149,7 +149,7 @@ You may set this visibility window on a per queue basis. For example, to set the visibility to 15 seconds: ``` -var queue = mongoDbQueue(db, 'queue', { visibility : 15 }) +var queue = new MongoDbQueue(db, 'queue', { visibility : 15 }) ``` All messages in this queue now have a visibility window of 15s, instead of the @@ -167,7 +167,7 @@ retrieval 10s after being added. To delay all messages by 10 seconds, try this: ``` -var queue = mongoDbQueue(db, 'queue', { delay : 10 }) +var queue = new MongoDbQueue(db, 'queue', { delay : 10 }) ``` This is now the default for every message added to the queue. @@ -182,8 +182,8 @@ automatically see problem messages. Pass in a queue (that you created) onto which these messages will be pushed: ```js -var deadQueue = mongoDbQueue(db, 'dead-queue') -var queue = mongoDbQueue(db, 'queue', { deadQueue : deadQueue }) +var deadQueue = new MongoDbQueue(db, 'dead-queue') +var queue = new MongoDbQueue(db, 'queue', { deadQueue : deadQueue }) ``` If you pop a message off the `queue` over `maxRetries` times and still have not acked it, @@ -246,7 +246,7 @@ If you want to opt in to using the newer `returnDocument`, set the `returnDocume to `true`: ``` -var queue = mongoDbQueue(db, 'queue', { returnDocument : true }) +var queue = new MongoDbQueue(db, 'queue', { returnDocument : true }) ``` ## Operations ## diff --git a/mongodb-queue.js b/mongodb-queue.js index 6e5f1b0..fb781d7 100644 --- a/mongodb-queue.js +++ b/mongodb-queue.js @@ -10,32 +10,27 @@ * **/ -var crypto = require('crypto') +const crypto = require('crypto') -// some helper functions function id() { - return crypto.randomBytes(16).toString('hex') + return crypto.randomBytes(16).toString('hex') } function now() { - return (new Date()).toISOString() + return (new Date()).toISOString() } function nowPlusSecs(secs) { - return (new Date(Date.now() + secs * 1000)).toISOString() + return (new Date(Date.now() + secs * 1000)).toISOString() } -module.exports = function(db, name, opts) { - return new Queue(db, name, opts) -} - -// the Queue object itself -function Queue(db, name, opts) { - if ( !db ) { - throw new Error("mongodb-queue: provide a mongodb.MongoClient.db") +module.exports = class Queue { + constructor(db, name, opts) { + if (!db) { + throw new Error("mongodb-queue: provide a mongodb.MongoClient.db") } - if ( !name ) { - throw new Error("mongodb-queue: provide a queue name") + if (!name) { + throw new Error("mongodb-queue: provide a queue name") } opts = opts || {} @@ -45,242 +40,179 @@ function Queue(db, name, opts) { this.visibility = opts.visibility || 30 this.delay = opts.delay || 0 - if ( opts.deadQueue ) { - this.deadQueue = opts.deadQueue - this.maxRetries = opts.maxRetries || 5 + if (opts.deadQueue) { + this.deadQueue = opts.deadQueue + this.maxRetries = opts.maxRetries || 5 } -} - -Queue.prototype.createIndexes = function(callback) { - var self = this + } - self.col.createIndex({ deleted : 1, visible : 1 }, function(err, indexname) { - if (err) return callback(err) - self.col.createIndex({ ack : 1 }, { unique : true, sparse : true }, function(err) { - if (err) return callback(err) - self.col.createIndex({ deleted : 1 }, { sparse : true }, function(err) { - if (err) return callback(err) - callback(null, indexname) - }) - }) - }) -} + async createIndexes() { + await Promise.all([ + this.col.createIndex({deleted: 1, visible: 1}), + this.col.createIndex({ack: 1}, {unique: true, sparse: true}), + this.col.createIndex({deleted: 1}, {sparse: true}) + ]) + } -Queue.prototype.add = function(payload, opts, callback) { - var self = this - if ( !callback ) { - callback = opts - opts = {} - } - var delay = opts.delay || self.delay - var visible = delay ? nowPlusSecs(delay) : now() + async add(payload, opts = {}) { + const delay = opts.delay || this.delay + const visible = delay ? nowPlusSecs(delay) : now() - var msgs = [] + const msgs = [] if (payload instanceof Array) { - if (payload.length === 0) { - var errMsg = 'Queue.add(): Array payload length must be greater than 0' - return callback(new Error(errMsg)) - } - payload.forEach(function(payload) { - msgs.push({ - visible : visible, - payload : payload, - }) - }) - } else { + if (payload.length === 0) { + throw new Error('Queue.add(): Array payload length must be greater than 0') + } + payload.forEach(function (payload) { msgs.push({ - visible : visible, - payload : payload, + visible: visible, + payload: payload, }) + }) + } else { + msgs.push({ + visible: visible, + payload: payload, + }) } - self.col.insertMany(msgs, function(err, results) { - if (err) return callback(err) - if (payload instanceof Array) return callback(null, '' + results.insertedIds) - callback(null, '' + results.insertedIds[0]) - }) -} - -Queue.prototype.get = function(opts, callback) { - var self = this - if ( !callback ) { - callback = opts - opts = {} - } + const results = await this.col.insertMany(msgs) + if (payload instanceof Array) return '' + results.insertedIds + return '' + results.insertedIds[0] + } - var visibility = opts.visibility || self.visibility - var query = { - deleted : null, - visible : { $lte : now() }, + async get(opts = {}) { + const visibility = opts.visibility || this.visibility + const query = { + deleted: null, + visible: {$lte: now()}, } - var sort = { - _id : 1 + const sort = { + _id: 1 } - var update = { - $inc : { tries : 1 }, - $set : { - ack : id(), - visible : nowPlusSecs(visibility), - } + const update = { + $inc: {tries: 1}, + $set: { + ack: id(), + visible: nowPlusSecs(visibility), + } } - var options = { - sort: sort, - returnDocument: 'after' + const options = { + sort: sort, + returnDocument: 'after' } - self.col.findOneAndUpdate(query, update, options, function(err, result) { - if (err) return callback(err) - var msg = result.value - if (!msg) return callback() - - // convert to an external representation - msg = { - // convert '_id' to an 'id' string - id : '' + msg._id, - ack : msg.ack, - payload : msg.payload, - tries : msg.tries, - } - // if we have a deadQueue, then check the tries, else don't - if ( self.deadQueue ) { - // check the tries - if ( msg.tries > self.maxRetries ) { - // So: - // 1) add this message to the deadQueue - // 2) ack this message from the regular queue - // 3) call ourself to return a new message (if exists) - self.deadQueue.add(msg, function(err) { - if (err) return callback(err) - self.ack(msg.ack, function(err) { - if (err) return callback(err) - self.get(callback) - }) - }) - return - } - } + const result = await this.col.findOneAndUpdate(query, update, options) + let msg = result.value + if (!msg) return - callback(null, msg) - }) -} + // convert to an external representation + msg = { + // convert '_id' to an 'id' string + id: '' + msg._id, + ack: msg.ack, + payload: msg.payload, + tries: msg.tries, + } -Queue.prototype.ping = function(ack, opts, callback) { - var self = this - if ( !callback ) { - callback = opts - opts = {} + // check the tries + if (this.deadQueue && msg.tries > this.maxRetries) { + // So: + // 1) add this message to the deadQueue + // 2) ack this message from the regular queue + // 3) call ourself to return a new message (if exists) + await this.deadQueue.add(msg) + await this.ack(msg.ack) + return this.get() } - var visibility = opts.visibility || self.visibility - var query = { - ack : ack, - visible : { $gt : now() }, - deleted : null, + return msg + } + + async ping(ack, opts = {}) { + const visibility = opts.visibility || this.visibility + const query = { + ack: ack, + visible: {$gt: now()}, + deleted: null, } - var update = { - $set : { - visible : nowPlusSecs(visibility) - } + const update = { + $set: { + visible: nowPlusSecs(visibility) + } } - var options = { - returnDocument: 'after' + const options = { + returnDocument: 'after' } if (opts.resetTries) { - update.$set.tries = 0 + update.$set.tries = 0 } - self.col.findOneAndUpdate(query, update, options, function(err, msg, blah) { - if (err) return callback(err) - if ( !msg.value ) { - return callback(new Error("Queue.ping(): Unidentified ack : " + ack)) - } - callback(null, '' + msg.value._id) - }) -} - -Queue.prototype.ack = function(ack, callback) { - var self = this + const msg = await this.col.findOneAndUpdate(query, update, options) + if (!msg.value) { + throw new Error("Queue.ping(): Unidentified ack : " + ack) + } + return '' + msg.value._id + } - var query = { - ack : ack, - visible : { $gt : now() }, - deleted : null, + async ack(ack) { + const query = { + ack: ack, + visible: {$gt: now()}, + deleted: null, } - var update = { - $set : { - deleted : now(), - } + const update = { + $set: { + deleted: now(), + } } - var options = { - returnDocument: 'after' + const options = { + returnDocument: 'after' } - self.col.findOneAndUpdate(query, update, options, function(err, msg) { - if (err) return callback(err) - if ( !msg.value ) { - return callback(new Error("Queue.ack(): Unidentified ack : " + ack)) - } - callback(null, '' + msg.value._id) - }) -} - -Queue.prototype.clean = function(callback) { - var self = this - - var query = { - deleted : { $exists : true }, + const msg = await this.col.findOneAndUpdate(query, update, options) + if (!msg.value) { + throw new Error("Queue.ack(): Unidentified ack : " + ack) } + return '' + msg.value._id + } - self.col.deleteMany(query, callback) -} - -Queue.prototype.total = function(callback) { - var self = this + async clean() { + const query = { + deleted: {$exists: true}, + } - self.col.countDocuments(function(err, count) { - if (err) return callback(err) - callback(null, count) - }) -} + return this.col.deleteMany(query) + } -Queue.prototype.size = function(callback) { - var self = this + async total() { + return this.col.countDocuments() + } - var query = { - deleted : null, - visible : { $lte : now() }, + async size() { + const query = { + deleted: null, + visible: {$lte: now()}, } - self.col.countDocuments(query, function(err, count) { - if (err) return callback(err) - callback(null, count) - }) -} + return this.col.countDocuments(query) + } -Queue.prototype.inFlight = function(callback) { - var self = this - - var query = { - ack : { $exists : true }, - visible : { $gt : now() }, - deleted : null, + async inFlight() { + const query = { + ack: {$exists: true}, + visible: {$gt: now()}, + deleted: null, } - self.col.countDocuments(query, function(err, count) { - if (err) return callback(err) - callback(null, count) - }) -} - -Queue.prototype.done = function(callback) { - var self = this + return this.col.countDocuments(query) + } - var query = { - deleted : { $exists : true }, + async done() { + const query = { + deleted: {$exists: true}, } - self.col.countDocuments(query, function(err, count) { - if (err) return callback(err) - callback(null, count) - }) + return this.col.countDocuments(query) + } } diff --git a/package.json b/package.json index 6fe3e12..a0cd8a7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@reedsy/mongodb-queue", - "version": "4.0.0-reedsy-2.0.1", + "version": "4.0.0-reedsy-3.0.0", "description": "Message queues which uses MongoDB.", "main": "mongodb-queue.js", "scripts": { @@ -8,12 +8,11 @@ }, "dependencies": {}, "devDependencies": { - "async": "^2.6.2", - "mongodb": "^4.0.0", + "mongodb": "^5.0.0", "tape": "^4.10.1" }, "peerDependencies": { - "mongodb": "^4.0.0" + "mongodb": "^4.0.0 || ^5.0.0" }, "homepage": "https://github.com/chilts/mongodb-queue", "repository": { diff --git a/test/_timeout.js b/test/_timeout.js new file mode 100644 index 0000000..2ba20bc --- /dev/null +++ b/test/_timeout.js @@ -0,0 +1,7 @@ +function timeout(millis) { + return new Promise((resolve) => setTimeout(resolve, millis)) +} + +module.exports = { + timeout, +} diff --git a/test/clean.js b/test/clean.js index 9f56d54..1624e65 100644 --- a/test/clean.js +++ b/test/clean.js @@ -1,166 +1,41 @@ -var async = require('async') -var test = require('tape') +const test = require('tape') -var setup = require('./setup.js') -var mongoDbQueue = require('../') +const setup = require('./setup.js') +const MongoDbQueue = require('../') -setup(function(client, db) { +setup().then(({client, db}) => { - test('clean: check deleted messages are deleted', function(t) { - var queue = mongoDbQueue(db, 'clean', { visibility : 3 }) - var msg + test('clean: check deleted messages are deleted', async function (t) { + const q = new MongoDbQueue(db, 'clean', {visibility: 3}) + let msg + let id - async.series( - [ - function(next) { - queue.size(function(err, size) { - t.ok(!err, 'There is no error.') - t.equal(size, 0, 'There is currently nothing on the queue') - next() - }) - }, - function(next) { - queue.total(function(err, size) { - t.ok(!err, 'There is no error.') - t.equal(size, 0, 'There is currently nothing in the queue at all') - next() - }) - }, - function(next) { - queue.clean(function(err) { - t.ok(!err, 'There is no error.') - next() - }) - }, - function(next) { - queue.size(function(err, size) { - t.ok(!err, 'There is no error.') - t.equal(size, 0, 'There is currently nothing on the queue') - next() - }) - }, - function(next) { - queue.total(function(err, size) { - t.ok(!err, 'There is no error.') - t.equal(size, 0, 'There is currently nothing in the queue at all') - next() - }) - }, - function(next) { - queue.add('Hello, World!', function(err) { - t.ok(!err, 'There is no error when adding a message.') - next() - }) - }, - function(next) { - queue.clean(function(err) { - t.ok(!err, 'There is no error.') - next() - }) - }, - function(next) { - queue.size(function(err, size) { - t.ok(!err, 'There is no error.') - t.equal(size, 1, 'Queue size is correct') - next() - }) - }, - function(next) { - queue.total(function(err, size) { - t.ok(!err, 'There is no error.') - t.equal(size, 1, 'Queue total is correct') - next() - }) - }, - function(next) { - queue.get(function(err, newMsg) { - msg = newMsg - t.ok(msg.id, 'Got a msg.id (sanity check)') - next() - }) - }, - function(next) { - queue.size(function(err, size) { - t.ok(!err, 'There is no error.') - t.equal(size, 0, 'Queue size is correct') - next() - }) - }, - function(next) { - queue.total(function(err, size) { - t.ok(!err, 'There is no error.') - t.equal(size, 1, 'Queue total is correct') - next() - }) - }, - function(next) { - queue.clean(function(err) { - t.ok(!err, 'There is no error.') - next() - }) - }, - function(next) { - queue.size(function(err, size) { - t.ok(!err, 'There is no error.') - t.equal(size, 0, 'Queue size is correct') - next() - }) - }, - function(next) { - queue.total(function(err, size) { - t.ok(!err, 'There is no error.') - t.equal(size, 1, 'Queue total is correct') - next() - }) - }, - function(next) { - queue.ack(msg.ack, function(err, id) { - t.ok(!err, 'No error when acking the message') - t.ok(id, 'Received an id when acking this message') - next() - }) - }, - function(next) { - queue.size(function(err, size) { - t.ok(!err, 'There is no error.') - t.equal(size, 0, 'Queue size is correct') - next() - }) - }, - function(next) { - queue.total(function(err, size) { - t.ok(!err, 'There is no error.') - t.equal(size, 1, 'Queue total is correct') - next() - }) - }, - function(next) { - queue.clean(function(err) { - t.ok(!err, 'There is no error.') - next() - }) - }, - function(next) { - queue.size(function(err, size) { - t.ok(!err, 'There is no error.') - t.equal(size, 0, 'Queue size is correct') - next() - }) - }, - function(next) { - queue.total(function(err, size) { - t.ok(!err, 'There is no error.') - t.equal(size, 0, 'Queue total is correct') - next() - }) - }, - ], - function(err) { - if (err) t.fail(err) - t.pass('Finished test ok') - t.end() - } - ) + t.equal(await q.size(), 0, 'There is currently nothing on the queue') + t.equal(await q.total(), 0, 'There is currently nothing in the queue at all') + await q.clean() + t.equal(await q.size(), 0, 'There is currently nothing on the queue') + t.equal(await q.total(), 0, 'There is currently nothing in the queue at all') + await q.add('Hello, World!') + await q.clean() + t.equal(await q.size(), 1, 'Queue size is correct') + t.equal(await q.total(), 1, 'Queue total is correct') + msg = await q.get() + t.ok(msg.id, 'Got a msg.id (sanity check)') + t.equal(await q.size(), 0, 'Queue size is correct') + t.equal(await q.total(), 1, 'Queue total is correct') + await q.clean() + t.equal(await q.size(), 0, 'Queue size is correct') + t.equal(await q.total(), 1, 'Queue total is correct') + id = await q.ack(msg.ack) + t.ok(id, 'Received an id when acking this message') + t.equal(await q.size(), 0, 'Queue size is correct') + t.equal(await q.total(), 1, 'Queue total is correct') + await q.clean() + t.equal(await q.size(), 0, 'Queue size is correct') + t.equal(await q.total(), 0, 'Queue total is correct') + + t.pass('Finished test ok') + t.end() }) test('client.close()', function(t) { diff --git a/test/dead-queue.js b/test/dead-queue.js index 8c4a38b..13286c4 100644 --- a/test/dead-queue.js +++ b/test/dead-queue.js @@ -1,178 +1,80 @@ -var async = require('async') -var test = require('tape') +const test = require('tape') -var setup = require('./setup.js') -var mongoDbQueue = require('../') +const setup = require('./setup.js') +const MongoDbQueue = require('../') -setup(function(client, db) { +setup().then(({client, db}) => { - test('first test', function(t) { - var queue = mongoDbQueue(db, 'queue', { visibility : 3, deadQueue : 'dead-queue' }) + test('first test', function (t) { + const queue = new MongoDbQueue(db, 'queue', {visibility: 3, deadQueue: 'dead-queue'}) t.ok(queue, 'Queue created ok') t.end() }); - test('single message going over 5 tries, should appear on dead-queue', function(t) { - var deadQueue = mongoDbQueue(db, 'dead-queue') - var queue = mongoDbQueue(db, 'queue', { visibility : 1, deadQueue : deadQueue }) - var msg - var origId - - async.series( - [ - function(next) { - queue.add('Hello, World!', function(err, id) { - t.ok(!err, 'There is no error when adding a message.') - t.ok(id, 'Received an id for this message') - origId = id - next() - }) - }, - function(next) { - queue.get(function(err, thisMsg) { - setTimeout(function() { - t.pass('First expiration') - next() - }, 2 * 1000) - }) - }, - function(next) { - queue.get(function(err, thisMsg) { - setTimeout(function() { - t.pass('Second expiration') - next() - }, 2 * 1000) - }) - }, - function(next) { - queue.get(function(err, thisMsg) { - setTimeout(function() { - t.pass('Third expiration') - next() - }, 2 * 1000) - }) - }, - function(next) { - queue.get(function(err, thisMsg) { - setTimeout(function() { - t.pass('Fourth expiration') - next() - }, 2 * 1000) - }) - }, - function(next) { - queue.get(function(err, thisMsg) { - setTimeout(function() { - t.pass('Fifth expiration') - next() - }, 2 * 1000) - }) - }, - function(next) { - queue.get(function(err, id) { - t.ok(!err, 'No error when getting no messages') - t.ok(!msg, 'No msg received') - next() - }) - }, - function(next) { - deadQueue.get(function(err, msg) { - t.ok(!err, 'No error when getting from the deadQueue') - t.ok(msg.id, 'Got a message id from the deadQueue') - t.equal(msg.payload.id, origId, 'Got the same message id as the original message') - t.equal(msg.payload.payload, 'Hello, World!', 'Got the same as the original message') - t.equal(msg.payload.tries, 6, 'Got the tries as 6') - next() - }) - }, - ], - function(err) { - t.ok(!err, 'No error during single round-trip test') - t.end() - } - ) + test('single message going over 5 tries, should appear on dead-queue', async function (t) { + const deadQueue = new MongoDbQueue(db, 'dead-queue') + const queue = new MongoDbQueue(db, 'queue', {visibility: 1, deadQueue: deadQueue}) + let msg + let origId + + origId = await queue.add('Hello, World!') + t.ok(origId, 'Received an id for this message') + + await queue.get() + + for (let i = 1; i <= 5; i++) { + await queue.get() + await new Promise((resolve) => setTimeout(function () { + t.pass(`Expiration #${i}`) + resolve() + }, 2 * 1000)) + } + + msg = await queue.get() + t.ok(!msg, 'No msg received') + + msg = await deadQueue.get() + t.ok(msg.id, 'Got a message id from the deadQueue') + t.equal(msg.payload.id, origId, 'Got the same message id as the original message') + t.equal(msg.payload.payload, 'Hello, World!', 'Got the same as the original message') + t.equal(msg.payload.tries, 6, 'Got the tries as 6') + + t.end() }) - test('two messages, with first going over 3 tries', function(t) { - var deadQueue = mongoDbQueue(db, 'dead-queue-2') - var queue = mongoDbQueue(db, 'queue-2', { visibility : 1, deadQueue : deadQueue, maxRetries : 3 }) - var msg - var origId, origId2 - - async.series( - [ - function(next) { - queue.add('Hello, World!', function(err, id) { - t.ok(!err, 'There is no error when adding a message.') - t.ok(id, 'Received an id for this message') - origId = id - next() - }) - }, - function(next) { - queue.add('Part II', function(err, id) { - t.ok(!err, 'There is no error when adding another message.') - t.ok(id, 'Received an id for this message') - origId2 = id - next() - }) - }, - function(next) { - queue.get(function(err, thisMsg) { - t.equal(thisMsg.id, origId, 'We return the first message on first go') - setTimeout(function() { - t.pass('First expiration') - next() - }, 2 * 1000) - }) - }, - function(next) { - queue.get(function(err, thisMsg) { - t.equal(thisMsg.id, origId, 'We return the first message on second go') - setTimeout(function() { - t.pass('Second expiration') - next() - }, 2 * 1000) - }) - }, - function(next) { - queue.get(function(err, thisMsg) { - t.equal(thisMsg.id, origId, 'We return the first message on third go') - setTimeout(function() { - t.pass('Third expiration') - next() - }, 2 * 1000) - }) - }, - function(next) { - // This is the 4th time, so we SHOULD have moved it to the dead queue - // pior to it being returned. - queue.get(function(err, msg) { - t.ok(!err, 'No error when getting the 2nd message') - t.equal(msg.id, origId2, 'Got the ID of the 2nd message') - t.equal(msg.payload, 'Part II', 'Got the same payload as the 2nd message') - next() - }) - }, - function(next) { - deadQueue.get(function(err, msg) { - t.ok(!err, 'No error when getting from the deadQueue') - t.ok(msg.id, 'Got a message id from the deadQueue') - t.equal(msg.payload.id, origId, 'Got the same message id as the original message') - t.equal(msg.payload.payload, 'Hello, World!', 'Got the same as the original message') - t.equal(msg.payload.tries, 4, 'Got the tries as 4') - next() - }) - }, - ], - function(err) { - t.ok(!err, 'No error during single round-trip test') - t.end() - } - ) + test('two messages, with first going over 3 tries', async function (t) { + const deadQueue = new MongoDbQueue(db, 'dead-queue-2') + const queue = new MongoDbQueue(db, 'queue-2', {visibility: 1, deadQueue: deadQueue, maxRetries: 3}) + let msg + let origId, origId2 + + origId = await queue.add('Hello, World!') + t.ok(origId, 'Received an id for this message') + origId2 = await queue.add('Part II') + t.ok(origId2, 'Received an id for this message') + + for (let i = 1; i <= 3; i++) { + msg = await queue.get() + t.equal(msg.id, origId, 'We return the first message on first go') + await new Promise((resolve) => setTimeout(function () { + t.pass(`Expiration #${i}`) + resolve() + }, 2 * 1000)) + } + + msg = await queue.get() + t.equal(msg.id, origId2, 'Got the ID of the 2nd message') + t.equal(msg.payload, 'Part II', 'Got the same payload as the 2nd message') + + msg = await deadQueue.get() + t.ok(msg.id, 'Got a message id from the deadQueue') + t.equal(msg.payload.id, origId, 'Got the same message id as the original message') + t.equal(msg.payload.payload, 'Hello, World!', 'Got the same as the original message') + t.equal(msg.payload.tries, 4, 'Got the tries as 4') + t.end() }) - test('client.close()', function(t) { + test('client.close()', function (t) { t.pass('client.close()') client.close() t.end() diff --git a/test/default.js b/test/default.js index 460f46d..67a8d37 100644 --- a/test/default.js +++ b/test/default.js @@ -1,110 +1,65 @@ -var async = require('async') -var test = require('tape') +const test = require('tape') -var setup = require('./setup.js') -var mongoDbQueue = require('../') +const setup = require('./setup.js') +const MongoDbQueue = require('../') -setup(function(client, db) { +setup().then(({client, db}) => { - test('first test', function(t) { - var queue = mongoDbQueue(db, 'default') + test('first test', function (t) { + const queue = new MongoDbQueue(db, 'default') t.ok(queue, 'Queue created ok') t.end() }); - test('single round trip', function(t) { - var queue = mongoDbQueue(db, 'default') - var msg + test('single round trip', async function (t) { + const queue = new MongoDbQueue(db, 'default') + let msg + let id - async.series( - [ - function(next) { - queue.add('Hello, World!', function(err, id) { - t.ok(!err, 'There is no error when adding a message.') - t.ok(id, 'Received an id for this message') - next() - }) - }, - function(next) { - queue.get(function(err, thisMsg) { - console.log(thisMsg) - msg = thisMsg - t.ok(msg.id, 'Got a msg.id') - t.equal(typeof msg.id, 'string', 'msg.id is a string') - t.ok(msg.ack, 'Got a msg.ack') - t.equal(typeof msg.ack, 'string', 'msg.ack is a string') - t.ok(msg.tries, 'Got a msg.tries') - t.equal(typeof msg.tries, 'number', 'msg.tries is a number') - t.equal(msg.tries, 1, 'msg.tries is currently one') - t.equal(msg.payload, 'Hello, World!', 'Payload is correct') - next() - }) - }, - function(next) { - queue.ack(msg.ack, function(err, id) { - t.ok(!err, 'No error when acking the message') - t.ok(id, 'Received an id when acking this message') - next() - }) - }, - ], - function(err) { - t.ok(!err, 'No error during single round-trip test') - t.end() - } - ) + id = await queue.add('Hello, World!') + t.ok(id, 'Received an id for this message') + + msg = await queue.get() + t.ok(msg.id, 'Got a msg.id') + t.equal(typeof msg.id, 'string', 'msg.id is a string') + t.ok(msg.ack, 'Got a msg.ack') + t.equal(typeof msg.ack, 'string', 'msg.ack is a string') + t.ok(msg.tries, 'Got a msg.tries') + t.equal(typeof msg.tries, 'number', 'msg.tries is a number') + t.equal(msg.tries, 1, 'msg.tries is currently one') + t.equal(msg.payload, 'Hello, World!', 'Payload is correct') + + id = await queue.ack(msg.ack) + t.ok(id, 'Received an id when acking this message') + t.end() }) - test("single round trip, can't be acked again", function(t) { - var queue = mongoDbQueue(db, 'default') - var msg + test("single round trip, can't be acked again", async function (t) { + const queue = new MongoDbQueue(db, 'default') + let msg + let id - async.series( - [ - function(next) { - queue.add('Hello, World!', function(err, id) { - t.ok(!err, 'There is no error when adding a message.') - t.ok(id, 'Received an id for this message') - next() - }) - }, - function(next) { - queue.get(function(err, thisMsg) { - msg = thisMsg - t.ok(msg.id, 'Got a msg.id') - t.equal(typeof msg.id, 'string', 'msg.id is a string') - t.ok(msg.ack, 'Got a msg.ack') - t.equal(typeof msg.ack, 'string', 'msg.ack is a string') - t.ok(msg.tries, 'Got a msg.tries') - t.equal(typeof msg.tries, 'number', 'msg.tries is a number') - t.equal(msg.tries, 1, 'msg.tries is currently one') - t.equal(msg.payload, 'Hello, World!', 'Payload is correct') - next() - }) - }, - function(next) { - queue.ack(msg.ack, function(err, id) { - t.ok(!err, 'No error when acking the message') - t.ok(id, 'Received an id when acking this message') - next() - }) - }, - function(next) { - queue.ack(msg.ack, function(err, id) { - t.ok(err, 'There is an error when acking the message again') - t.ok(!id, 'No id received when trying to ack an already deleted message') - next() - }) - }, - ], - function(err) { - t.ok(!err, 'No error during single round-trip when trying to double ack') - t.end() - } - ) + id = await queue.add('Hello, World!') + t.ok(id, 'Received an id for this message') + msg = await queue.get() + t.ok(msg.id, 'Got a msg.id') + t.equal(typeof msg.id, 'string', 'msg.id is a string') + t.ok(msg.ack, 'Got a msg.ack') + t.equal(typeof msg.ack, 'string', 'msg.ack is a string') + t.ok(msg.tries, 'Got a msg.tries') + t.equal(typeof msg.tries, 'number', 'msg.tries is a number') + t.equal(msg.tries, 1, 'msg.tries is currently one') + t.equal(msg.payload, 'Hello, World!', 'Payload is correct') + id = await queue.ack(msg.ack) + t.ok(id, 'Received an id when acking this message') + id = await queue.ack(msg.ack) + .catch((err) => t.ok(err, 'There is an error when acking the message again')) + + t.ok(!id, 'No id received when trying to ack an already deleted message') + t.end() }) - test('client.close()', function(t) { + test('client.close()', function (t) { t.pass('client.close()') client.close() t.end() diff --git a/test/delay.js b/test/delay.js index d1c74ef..61a4efa 100644 --- a/test/delay.js +++ b/test/delay.js @@ -1,104 +1,57 @@ -var async = require('async') -var test = require('tape') - -var setup = require('./setup.js') -var mongoDbQueue = require('../') - -setup(function(client, db) { - - test('delay: check messages on this queue are returned after the delay', function(t) { - var queue = mongoDbQueue(db, 'delay', { delay : 3 }) - - async.series( - [ - function(next) { - queue.add('Hello, World!', function(err, id) { - t.ok(!err, 'There is no error when adding a message.') - t.ok(id, 'There is an id returned when adding a message.') - next() - }) - }, - function(next) { - // get something now and it shouldn't be there - queue.get(function(err, msg) { - t.ok(!err, 'No error when getting no messages') - t.ok(!msg, 'No msg received') - // now wait 4s - setTimeout(next, 4 * 1000) - }) - }, - function(next) { - // get something now and it SHOULD be there - queue.get(function(err, msg) { - t.ok(!err, 'No error when getting a message') - t.ok(msg.id, 'Got a message id now that the message delay has passed') - queue.ack(msg.ack, next) - }) - }, - function(next) { - queue.get(function(err, msg) { - // no more messages - t.ok(!err, 'No error when getting no messages') - t.ok(!msg, 'No more messages') - next() - }) - }, - ], - function(err) { - if (err) t.fail(err) - t.pass('Finished test ok') - t.end() - } - ) +const test = require('tape') + +const setup = require('./setup.js') +const MongoDbQueue = require('../') +const {timeout} = require('./_timeout.js') + +setup().then(({client, db}) => { + + test('delay: check messages on this queue are returned after the delay', async function (t) { + const queue = new MongoDbQueue(db, 'delay', {delay: 3}) + let id + let msg + + id = await queue.add('Hello, World!') + t.ok(id, 'There is an id returned when adding a message.') + // get something now and it shouldn't be there + msg = await queue.get() + t.ok(!msg, 'No msg received') + await timeout(4_000) + // get something now and it SHOULD be there + msg = await queue.get() + t.ok(msg.id, 'Got a message id now that the message delay has passed') + await queue.ack(msg.ack) + msg = await queue.get() + // no more messages + t.ok(!msg, 'No more messages') + t.pass('Finished test ok') + t.end() }) - test('delay: check an individual message delay overrides the queue delay', function(t) { - var queue = mongoDbQueue(db, 'delay') - - async.series( - [ - function(next) { - queue.add('I am delayed by 3 seconds', { delay : 3 }, function(err, id) { - t.ok(!err, 'There is no error when adding a message.') - t.ok(id, 'There is an id returned when adding a message.') - next() - }) - }, - function(next) { - // get something now and it shouldn't be there - queue.get(function(err, msg) { - t.ok(!err, 'No error when getting no messages') - t.ok(!msg, 'No msg received') - // now wait 4s - setTimeout(next, 4 * 1000) - }) - }, - function(next) { - // get something now and it SHOULD be there - queue.get(function(err, msg) { - t.ok(!err, 'No error when getting a message') - t.ok(msg.id, 'Got a message id now that the message delay has passed') - queue.ack(msg.ack, next) - }) - }, - function(next) { - queue.get(function(err, msg) { - // no more messages - t.ok(!err, 'No error when getting no messages') - t.ok(!msg, 'No more messages') - next() - }) - }, - ], - function(err) { - if (err) t.fail(err) - t.pass('Finished test ok') - t.end() - } - ) + test('delay: check an individual message delay overrides the queue delay', async function (t) { + const queue = new MongoDbQueue(db, 'delay') + let id + let msg + + id = await queue.add('I am delayed by 3 seconds', {delay: 3}) + t.ok(id, 'There is an id returned when adding a message.') + // get something now and it shouldn't be there + msg = await queue.get() + t.ok(!msg, 'No msg received') + await timeout(4_000) + // get something now and it SHOULD be there + msg = await queue.get() + t.ok(msg.id, 'Got a message id now that the message delay has passed') + await queue.ack(msg.ack) + msg = await queue.get() + // no more messages + t.ok(!msg, 'No more messages') + + t.pass('Finished test ok') + t.end() }) - test('client.close()', function(t) { + test('client.close()', function (t) { t.pass('client.close()') client.close() t.end() diff --git a/test/indexes.js b/test/indexes.js index 5724afc..e250b7c 100644 --- a/test/indexes.js +++ b/test/indexes.js @@ -1,21 +1,16 @@ -var async = require('async') -var test = require('tape') +const test = require('tape') -var setup = require('./setup.js') -var mongoDbQueue = require('../') +const setup = require('./setup.js') +const MongoDbQueue = require('../') -setup(function(client, db) { +setup().then(({client, db}) => { - test('visibility: check message is back in queue after 3s', function(t) { - t.plan(2) + test('visibility: check message is back in queue after 3s', async function(t) { + const queue = new MongoDbQueue(db, 'visibility', { visibility : 3 }) - var queue = mongoDbQueue(db, 'visibility', { visibility : 3 }) - - queue.createIndexes(function(err, indexName) { - t.ok(!err, 'There was no error when running .ensureIndexes()') - t.ok(indexName, 'receive indexName we created') - t.end() - }) + await queue.createIndexes() + t.pass('Indexes created') + t.end() }) test('client.close()', function(t) { diff --git a/test/many.js b/test/many.js index b95e907..39c7659 100644 --- a/test/many.js +++ b/test/many.js @@ -1,80 +1,50 @@ -var async = require('async') -var test = require('tape') +const test = require('tape') -var setup = require('./setup.js') -var mongoDbQueue = require('../') +const setup = require('./setup.js') +const MongoDbQueue = require('../') -var total = 250 +const total = 250 -setup(function(client, db) { +setup().then(({client, db}) => { - test('many: add ' + total + ' messages, get ' + total + ' back', function(t) { - var queue = mongoDbQueue(db, 'many') - var msgs = [] - var msgsToQueue = [] + test('many: add ' + total + ' messages, get ' + total + ' back', async function (t) { + const queue = new MongoDbQueue(db, 'many') + const msgs = [] + const msgsToQueue = [] - async.series( - [ - function(next) { - var i - for(i=0; i queue.ack(msg.ack)) ) + + t.pass('Acked all ' + total + ' messages') + t.pass('Finished test ok') + t.end() }) - test('many: add no messages, receive err in callback', function(t) { - var queue = mongoDbQueue(db, 'many') - var messages = [] - queue.add([], function(err) { - if (!err) t.fail('Error was not received') - t.pass('Finished test ok') - t.end() - }); + test('many: add no messages, receive err in callback', async function (t) { + const queue = new MongoDbQueue(db, 'many') + await queue.add([]) + .catch(() => t.pass('got error')) + t.end() }) - test('client.close()', function(t) { + test('client.close()', function (t) { t.pass('client.close()') client.close() t.end() diff --git a/test/multi.js b/test/multi.js index d3be71e..f34d6cb 100644 --- a/test/multi.js +++ b/test/multi.js @@ -1,71 +1,37 @@ -var async = require('async') -var test = require('tape') +const test = require('tape') -var setup = require('./setup.js') -var mongoDbQueue = require('../') +const setup = require('./setup.js') +const MongoDbQueue = require('../') -var total = 250 +const total = 250 -setup(function(client, db) { +setup().then(({client, db}) => { - test('multi: add ' + total + ' messages, get ' + total + ' back', function(t) { - var queue = mongoDbQueue(db, 'multi') - var msgs = [] + test('multi: add ' + total + ' messages, get ' + total + ' back', async function (t) { + const queue = new MongoDbQueue(db, 'multi') + const msgs = [] - async.series( - [ - function(next) { - var i, done = 0 - for(i=0; i queue.ack(msg.ack)) ) + + t.pass('Acked all ' + total + ' messages') + + t.end() }) - test('client.close()', function(t) { + test('client.close()', function (t) { t.pass('client.close()') client.close() t.end() diff --git a/test/ping.js b/test/ping.js index 5c61593..84cb5f4 100644 --- a/test/ping.js +++ b/test/ping.js @@ -1,235 +1,117 @@ -var async = require('async') -var test = require('tape') - -var setup = require('./setup.js') -var mongoDbQueue = require('../') - -setup(function(client, db) { - - test('ping: check a retrieved message with a ping can still be acked', function(t) { - var queue = mongoDbQueue(db, 'ping', { visibility : 5 }) - var msg - - async.series( - [ - function(next) { - queue.add('Hello, World!', function(err, id) { - t.ok(!err, 'There is no error when adding a message.') - t.ok(id, 'There is an id returned when adding a message.') - next() - }) - }, - function(next) { - // get something now and it shouldn't be there - queue.get(function(err, thisMsg) { - msg = thisMsg - t.ok(!err, 'No error when getting this message') - t.ok(msg.id, 'Got this message id') - // now wait 4s - setTimeout(next, 4 * 1000) - }) - }, - function(next) { - // ping this message so it will be kept alive longer, another 5s - queue.ping(msg.ack, function(err, id) { - t.ok(!err, 'No error when pinging a message') - t.ok(id, 'Received an id when acking this message') - // now wait 4s - setTimeout(next, 4 * 1000) - }) - }, - function(next) { - queue.ack(msg.ack, function(err, id) { - t.ok(!err, 'No error when acking this message') - t.ok(id, 'Received an id when acking this message') - next() - }) - }, - function(next) { - queue.get(function(err, msg) { - t.ok(!err, 'No error when getting no messages') - t.ok(!msg, 'No message when getting from an empty queue') - next() - }) - }, - ], - function(err) { - if (err) t.fail(err) - t.pass('Finished test ok') - t.end() - } - ) +const test = require('tape') +const {timeout} = require('./_timeout') + +const setup = require('./setup.js') +const MongoDbQueue = require('../') + +setup().then(({client, db}) => { + + test('ping: check a retrieved message with a ping can still be acked', async function (t) { + const queue = new MongoDbQueue(db, 'ping', {visibility: 5}) + let msg + let id + + id = await queue.add('Hello, World!') + t.ok(id, 'There is an id returned when adding a message.') + // get something now and it shouldn't be there + msg = await queue.get() + t.ok(msg.id, 'Got this message id') + await timeout(4_000) + // ping this message so it will be kept alive longer, another 5s + id = await queue.ping(msg.ack) + t.ok(id, 'Received an id when acking this message') + await timeout(4_000) + id = await queue.ack(msg.ack) + t.ok(id, 'Received an id when acking this message') + msg = await queue.get() + t.ok(!msg, 'No message when getting from an empty queue') + + t.pass('Finished test ok') + t.end() }) - test("ping: check that an acked message can't be pinged", function(t) { - var queue = mongoDbQueue(db, 'ping', { visibility : 5 }) - var msg - - async.series( - [ - function(next) { - queue.add('Hello, World!', function(err, id) { - t.ok(!err, 'There is no error when adding a message.') - t.ok(id, 'There is an id returned when adding a message.') - next() - }) - }, - function(next) { - // get something now and it shouldn't be there - queue.get(function(err, thisMsg) { - msg = thisMsg - t.ok(!err, 'No error when getting this message') - t.ok(msg.id, 'Got this message id') - next() - }) - }, - function(next) { - // ack the message - queue.ack(msg.ack, function(err, id) { - t.ok(!err, 'No error when acking this message') - t.ok(id, 'Received an id when acking this message') - next() - }) - }, - function(next) { - // ping this message, even though it has been acked - queue.ping(msg.ack, function(err, id) { - t.ok(err, 'Error when pinging an acked message') - t.ok(!id, 'Received no id when pinging an acked message') - next() - }) - }, - ], - function(err) { - if (err) t.fail(err) - t.pass('Finished test ok') - t.end() - } - ) + test("ping: check that an acked message can't be pinged", async function (t) { + const queue = new MongoDbQueue(db, 'ping', {visibility: 5}) + let msg + let id + + id = await queue.add('Hello, World!') + t.ok(id, 'There is an id returned when adding a message.') + // get something now and it shouldn't be there + msg = await queue.get() + t.ok(msg.id, 'Got this message id') + // ack the message + id = await queue.ack(msg.ack) + t.ok(id, 'Received an id when acking this message') + // ping this message, even though it has been acked + id = await queue.ping(msg.ack) + .catch((err) => t.ok(err, 'Error when pinging an acked message')) + t.ok(!id, 'Received no id when pinging an acked message') + + t.pass('Finished test ok') + t.end() }) -test("ping: check visibility option overrides the queue visibility", function(t) { - var queue = mongoDbQueue(db, 'ping', { visibility : 3 }) - var msg - - async.series( - [ - function(next) { - queue.add('Hello, World!', function(err, id) { - t.ok(!err, 'There is no error when adding a message.') - t.ok(id, 'There is an id returned when adding a message.') - next() - }) - }, - function(next) { - queue.get(function(err, thisMsg) { - msg = thisMsg - // message should reset in three seconds - t.ok(msg.id, 'Got a msg.id (sanity check)') - setTimeout(next, 2 * 1000) - }) - }, - function(next) { - // ping this message so it will be kept alive longer, another 5s instead of 3s - queue.ping(msg.ack, { visibility: 5 }, function(err, id) { - t.ok(!err, 'No error when pinging a message') - t.ok(id, 'Received an id when acking this message') - // wait 4s so the msg would normally have returns to the queue - setTimeout(next, 4 * 1000) - }) - }, - function(next) { - queue.get(function(err, msg) { - // messages should not be back yet - t.ok(!err, 'No error when getting no messages') - t.ok(!msg, 'No msg received') - // wait 2s so the msg should have returns to the queue - setTimeout(next, 2 * 1000) - }) - }, - function(next) { - queue.get(function(err, msg) { - // yes, there should be a message on the queue again - t.ok(msg.id, 'Got a msg.id (sanity check)') - queue.ack(msg.ack, function(err) { - t.ok(!err, 'No error when acking the message') - next() - }) - }) - }, - function(next) { - queue.get(function(err, msg) { - // no more messages - t.ok(!err, 'No error when getting no messages') - t.ok(!msg, 'No msg received') - next() - }) - } - ], - function(err) { - if (err) t.fail(err) - t.pass('Finished test ok') - t.end() - } - ) + test("ping: check visibility option overrides the queue visibility", async function (t) { + const queue = new MongoDbQueue(db, 'ping', {visibility: 3}) + let msg + let id + + id = await queue.add('Hello, World!') + t.ok(id, 'There is an id returned when adding a message.') + msg = await queue.get() + // message should reset in three seconds + t.ok(msg.id, 'Got a msg.id (sanity check)') + await timeout(2_000) + // ping this message so it will be kept alive longer, another 5s instead of 3s + id = await queue.ping(msg.ack, {visibility: 5}) + t.ok(id, 'Received an id when acking this message') + // wait 4s so the msg would normally have returns to the queue + await timeout(4_000) + msg = await queue.get() + // messages should not be back yet + t.ok(!msg, 'No msg received') + // wait 2s so the msg should have returns to the queue + await timeout(2_000) + msg = await queue.get() + // yes, there should be a message on the queue again + t.ok(msg.id, 'Got a msg.id (sanity check)') + await queue.ack(msg.ack) + msg = await queue.get() + // no more messages + t.ok(!msg, 'No msg received') + + t.pass('Finished test ok') + t.end() }) - test("ping: reset tries", function(t) { - var queue = mongoDbQueue(db, 'ping', { visibility: 3 }) - var msg - - async.series( - [ - function(next) { - queue.add('Hello, World!', function(err, id) { - t.ok(!err, 'There is no error when adding a message.') - t.ok(id, 'There is an id returned when adding a message.') - next() - }) - }, - function(next) { - queue.get(function(err, thisMsg) { - msg = thisMsg - // message should reset in three seconds - t.ok(msg.id, 'Got a msg.id (sanity check)') - setTimeout(next, 2 * 1000) - }) - }, - function(next) { - queue.ping(msg.ack, { resetTries: true }, function(err, id) { - t.ok(!err, 'No error when pinging a message') - t.ok(id, 'Received an id when acking this message') - // wait until the msg has returned to the queue - setTimeout(next, 6 * 1000) - }) - }, - function(next) { - queue.get(function(err, msg) { - t.equal(msg.tries, 1, 'Tries were reset') - queue.ack(msg.ack, function(err) { - t.ok(!err, 'No error when acking the message') - next() - }) - }) - }, - function(next) { - queue.get(function(err, msg) { - // no more messages - t.ok(!err, 'No error when getting no messages') - t.ok(!msg, 'No msg received') - next() - }) - } - ], - function(err) { - if (err) t.fail(err) - t.pass('Finished test ok') - t.end() - } - ) + test("ping: reset tries", async function (t) { + const queue = new MongoDbQueue(db, 'ping', {visibility: 3}) + let msg + let id + + id = await queue.add('Hello, World!') + t.ok(id, 'There is an id returned when adding a message.') + msg = await queue.get() + // message should reset in three seconds + t.ok(msg.id, 'Got a msg.id (sanity check)') + await timeout(2_000) + id = await queue.ping(msg.ack, {resetTries: true}) + t.ok(id, 'Received an id when acking this message') + // wait until the msg has returned to the queue + await timeout(6_000) + msg = await queue.get() + t.equal(msg.tries, 1, 'Tries were reset') + await queue.ack(msg.ack) + msg = await queue.get() + // no more messages + t.ok(!msg, 'No msg received') + + t.pass('Finished test ok') + t.end() }) - test('client.close()', function(t) { + test('client.close()', function (t) { t.pass('client.close()') client.close() t.end() diff --git a/test/setup.js b/test/setup.js index ea893bc..90cabe1 100644 --- a/test/setup.js +++ b/test/setup.js @@ -18,25 +18,14 @@ const collections = [ 'dead-queue-2', ] -module.exports = function(callback) { +module.exports = async function() { const client = new mongodb.MongoClient(url, { useNewUrlParser: true }) - client.connect(err => { - // we can throw since this is test-only - if (err) throw err - - const db = client.db(dbName) - - // empty out some collections to make sure there are no messages - let done = 0 - collections.forEach((col) => { - db.collection(col).deleteMany(() => { - done += 1 - if ( done === collections.length ) { - callback(client, db) - } - }) - }) - }) + await client.connect() + const db = client.db(dbName) + await Promise.all( + collections.map((col) => db.collection(col).deleteMany()) + ) + return {client, db} } diff --git a/test/stats.js b/test/stats.js index fab3fa0..6679321 100644 --- a/test/stats.js +++ b/test/stats.js @@ -1,204 +1,71 @@ -var async = require('async') -var test = require('tape') +const test = require('tape') +const {timeout} = require('./_timeout') -var setup = require('./setup.js') -var mongoDbQueue = require('../') +const setup = require('./setup.js') +const MongoDbQueue = require('../') -setup(function(client, db) { +setup().then(({client, db}) => { - test('first test', function(t) { - var queue = mongoDbQueue(db, 'stats') + test('first test', function (t) { + const queue = new MongoDbQueue(db, 'stats') t.ok(queue, 'Queue created ok') t.end() }); - test('stats for a single message added, received and acked', function(t) { - var queue = mongoDbQueue(db, 'stats1') - var msg + test('stats for a single message added, received and acked', async function (t) { + const q = new MongoDbQueue(db, 'stats1') + let msg + let id - async.series( - [ - function(next) { - queue.add('Hello, World!', function(err, id) { - t.ok(!err, 'There is no error when adding a message.') - t.ok(id, 'Received an id for this message') - next() - }) - }, - function(next) { - queue.total(function(err, count) { - t.equal(count, 1, 'Total number of messages is one') - next() - }) - }, - function(next) { - queue.size(function(err, count) { - t.equal(count, 1, 'Size of queue is one') - next() - }) - }, - function(next) { - queue.inFlight(function(err, count) { - t.equal(count, 0, 'There are no inFlight messages') - next() - }) - }, - function(next) { - queue.done(function(err, count) { - t.equal(count, 0, 'There are no done messages') - next() - }) - }, - function(next) { - // let's set one to be inFlight - queue.get(function(err, newMsg) { - msg = newMsg - next() - }) - }, - function(next) { - queue.total(function(err, count) { - t.equal(count, 1, 'Total number of messages is still one') - next() - }) - }, - function(next) { - queue.size(function(err, count) { - t.equal(count, 0, 'Size of queue is now zero (ie. none to come)') - next() - }) - }, - function(next) { - queue.inFlight(function(err, count) { - t.equal(count, 1, 'There is one inflight message') - next() - }) - }, - function(next) { - queue.done(function(err, count) { - t.equal(count, 0, 'There are still no done messages') - next() - }) - }, - function(next) { - // now ack that message - queue.ack(msg.ack, function(err, newMsg) { - msg = newMsg - next() - }) - }, - function(next) { - queue.total(function(err, count) { - t.equal(count, 1, 'Total number of messages is again one') - next() - }) - }, - function(next) { - queue.size(function(err, count) { - t.equal(count, 0, 'Size of queue is still zero (ie. none to come)') - next() - }) - }, - function(next) { - queue.inFlight(function(err, count) { - t.equal(count, 0, 'There are no inflight messages anymore') - next() - }) - }, - function(next) { - queue.done(function(err, count) { - t.equal(count, 1, 'There is now one processed message') - next() - }) - }, - ], - function(err) { - t.ok(!err, 'No error when doing stats on one message') - t.end() - } - ) + id = await q.add('Hello, World!') + t.ok(id, 'Received an id for this message') + t.equal(await q.total(), 1, 'Total number of messages is one') + t.equal(await q.size(), 1, 'Size of queue is one') + t.equal(await q.inFlight(), 0, 'There are no inFlight messages') + t.equal(await q.done(), 0, 'There are no done messages') + msg = await q.get() + t.equal(await q.total(), 1, 'Total number of messages is still one') + t.equal(await q.size(), 0, 'Size of queue is now zero (ie. none to come)') + t.equal(await q.inFlight(), 1, 'There is one inflight message') + t.equal(await q.done(), 0, 'There are still no done messages') + // now ack that message + msg = await q.ack(msg.ack) + t.equal(await q.total(), 1, 'Total number of messages is again one') + t.equal(await q.size(), 0, 'Size of queue is still zero (ie. none to come)') + t.equal(await q.inFlight(), 0, 'There are no inflight messages anymore') + t.equal(await q.done(), 1, 'There is now one processed message') + + t.end() }) // ToDo: add more tests for adding a message, getting it and letting it lapse // then re-checking all stats. - test('stats for a single message added, received, timed-out and back on queue', function(t) { - var queue = mongoDbQueue(db, 'stats2', { visibility : 3 }) + test('stats for a single message added, received, timed-out and back on queue', async function (t) { + const q = new MongoDbQueue(db, 'stats2', {visibility: 3}) + let id + let msg - async.series( - [ - function(next) { - queue.add('Hello, World!', function(err, id) { - t.ok(!err, 'There is no error when adding a message.') - t.ok(id, 'Received an id for this message') - next() - }) - }, - function(next) { - queue.total(function(err, count) { - t.equal(count, 1, 'Total number of messages is one') - next() - }) - }, - function(next) { - queue.size(function(err, count) { - t.equal(count, 1, 'Size of queue is one') - next() - }) - }, - function(next) { - queue.inFlight(function(err, count) { - t.equal(count, 0, 'There are no inFlight messages') - next() - }) - }, - function(next) { - queue.done(function(err, count) { - t.equal(count, 0, 'There are no done messages') - next() - }) - }, - function(next) { - // let's set one to be inFlight - queue.get(function(err, msg) { - // msg is ignored, we don't care about the message here - setTimeout(next, 4 * 1000) - }) - }, - function(next) { - queue.total(function(err, count) { - t.equal(count, 1, 'Total number of messages is still one') - next() - }) - }, - function(next) { - queue.size(function(err, count) { - t.equal(count, 1, 'Size of queue is still at one') - next() - }) - }, - function(next) { - queue.inFlight(function(err, count) { - t.equal(count, 0, 'There are no inflight messages again') - next() - }) - }, - function(next) { - queue.done(function(err, count) { - t.equal(count, 0, 'There are still no done messages') - next() - }) - }, - ], - function(err) { - t.ok(!err, 'No error when doing stats on one message') - t.end() - } - ) + id = await q.add('Hello, World!') + t.ok(id, 'Received an id for this message') + t.equal(await q.total(), 1, 'Total number of messages is one') + t.equal(await q.size(), 1, 'Size of queue is one') + t.equal(await q.inFlight(), 0, 'There are no inFlight messages') + t.equal(await q.done(), 0, 'There are no done messages') + // let's set one to be inFlight + msg = await q.get() + // msg is ignored, we don't care about the message here + await timeout(4_000) + t.equal(await q.total(), 1, 'Total number of messages is still one') + t.equal(await q.size(), 1, 'Size of queue is still at one') + t.equal(await q.inFlight(), 0, 'There are no inflight messages again') + t.equal(await q.done(), 0, 'There are still no done messages') + + t.end() }) - test('client.close()', function(t) { + test('client.close()', function (t) { t.pass('client.close()') client.close() t.end() diff --git a/test/visibility.js b/test/visibility.js index 56fda8c..5fa6153 100644 --- a/test/visibility.js +++ b/test/visibility.js @@ -1,171 +1,90 @@ -var async = require('async') -var test = require('tape') - -var setup = require('./setup.js') -var mongoDbQueue = require('../') - -setup(function(client, db) { - - test('visibility: check message is back in queue after 3s', function(t) { - var queue = mongoDbQueue(db, 'visibility', { visibility : 3 }) - - async.series( - [ - function(next) { - queue.add('Hello, World!', function(err) { - t.ok(!err, 'There is no error when adding a message.') - next() - }) - }, - function(next) { - queue.get(function(err, msg) { - // wait over 3s so the msg returns to the queue - t.ok(msg.id, 'Got a msg.id (sanity check)') - setTimeout(next, 4 * 1000) - }) - }, - function(next) { - queue.get(function(err, msg) { - // yes, there should be a message on the queue again - t.ok(msg.id, 'Got a msg.id (sanity check)') - queue.ack(msg.ack, function(err) { - t.ok(!err, 'No error when acking the message') - next() - }) - }) - }, - function(next) { - queue.get(function(err, msg) { - // no more messages - t.ok(!err, 'No error when getting no messages') - t.ok(!msg, 'No msg received') - next() - }) - }, - ], - function(err) { - if (err) t.fail(err) - t.pass('Finished test ok') - t.end() - } - ) +const test = require('tape') +const {timeout} = require('./_timeout') + +const setup = require('./setup.js') +const MongoDbQueue = require('../') + +setup().then(({client, db}) => { + + test('visibility: check message is back in queue after 3s', async function (t) { + const queue = new MongoDbQueue(db, 'visibility', {visibility: 3}) + let msg + + await queue.add('Hello, World!') + msg = await queue.get() + t.ok(msg.id, 'Got a msg.id (sanity check)') + await timeout(4_000) + msg = await queue.get() + // yes, there should be a message on the queue again + t.ok(msg.id, 'Got a msg.id (sanity check)') + await queue.ack(msg.ack) + msg = await queue.get() + t.ok(!msg, 'No msg received') + + t.pass('Finished test ok') + t.end() }) - test("visibility: check that a late ack doesn't remove the msg", function(t) { - var queue = mongoDbQueue(db, 'visibility', { visibility : 3 }) - var originalAck - - async.series( - [ - function(next) { - queue.add('Hello, World!', function(err) { - t.ok(!err, 'There is no error when adding a message.') - next() - }) - }, - function(next) { - queue.get(function(err, msg) { - t.ok(msg.id, 'Got a msg.id (sanity check)') - - // remember this original ack - originalAck = msg.ack - - // wait over 3s so the msg returns to the queue - setTimeout(function() { - t.pass('Back from timeout, now acking the message') - - // now ack the message but too late - it shouldn't be deleted - queue.ack(msg.ack, function(err, msg) { - t.ok(err, 'Got an error when acking the message late') - t.ok(!msg, 'No message was updated') - next() - }) - }, 4 * 1000) - }) - }, - function(next) { - queue.get(function(err, msg) { - // the message should now be able to be retrieved, with a new 'ack' id - t.ok(msg.id, 'Got a msg.id (sanity check)') - t.notEqual(msg.ack, originalAck, 'Original ack and new ack are different') - - // now ack this new retrieval - queue.ack(msg.ack, next) - }) - }, - function(next) { - queue.get(function(err, msg) { - // no more messages - t.ok(!err, 'No error when getting no messages') - t.ok(!msg, 'No msg received') - next() - }) - }, - ], - function(err) { - if (err) t.fail(err) - t.pass('Finished test ok') - t.end() - } - ) + test("visibility: check that a late ack doesn't remove the msg", async function (t) { + const queue = new MongoDbQueue(db, 'visibility', {visibility: 3}) + let originalAck + let msg + + await queue.add('Hello, World!') + msg = await queue.get() + t.ok(msg.id, 'Got a msg.id (sanity check)') + // remember this original ack + originalAck = msg.ack + // wait over 3s so the msg returns to the queue + await timeout(4_000) + + t.pass('Back from timeout, now acking the message') + + // now ack the message but too late - it shouldn't be deleted + msg = await queue.ack(msg.ack) + .catch((err) => t.ok(err, 'Got an error when acking the message late')) + t.ok(!msg, 'No message was updated') + msg = await queue.get() + // the message should now be able to be retrieved, with a new 'ack' id + t.ok(msg.id, 'Got a msg.id (sanity check)') + t.notEqual(msg.ack, originalAck, 'Original ack and new ack are different') + + // now ack this new retrieval + await queue.ack(msg.ack) + msg = await queue.get() + + // no more messages + t.ok(!msg, 'No msg received') + + t.pass('Finished test ok') + t.end() }) - test("visibility: check visibility option overrides the queue visibility", function(t) { - var queue = mongoDbQueue(db, 'visibility', { visibility : 2 }) - var originalAck - - async.series( - [ - function(next) { - queue.add('Hello, World!', function(err) { - t.ok(!err, 'There is no error when adding a message.') - next() - }) - }, - function(next) { - queue.get({ visibility: 4 }, function(err, msg) { - // wait over 2s so the msg would normally have returns to the queue - t.ok(msg.id, 'Got a msg.id (sanity check)') - setTimeout(next, 3 * 1000) - }) - }, - function(next) { - queue.get(function(err, msg) { - // messages should not be back yet - t.ok(!err, 'No error when getting no messages') - t.ok(!msg, 'No msg received') - // wait 2s so the msg should have returns to the queue - setTimeout(next, 2 * 1000) - }) - }, - function(next) { - queue.get(function(err, msg) { - // yes, there should be a message on the queue again - t.ok(msg.id, 'Got a msg.id (sanity check)') - queue.ack(msg.ack, function(err) { - t.ok(!err, 'No error when acking the message') - next() - }) - }) - }, - function(next) { - queue.get(function(err, msg) { - // no more messages - t.ok(!err, 'No error when getting no messages') - t.ok(!msg, 'No msg received') - next() - }) - } - ], - function(err) { - if (err) t.fail(err) - t.pass('Finished test ok') - t.end() - } - ) + test("visibility: check visibility option overrides the queue visibility", async function (t) { + const queue = new MongoDbQueue(db, 'visibility', {visibility: 2}) + let msg + + await queue.add('Hello, World!') + msg = await queue.get({visibility: 4}) + t.ok(msg.id, 'Got a msg.id (sanity check)') + // wait over 2s so the msg would normally have returns to the queue + await timeout(3_000) + msg = await queue.get() + t.ok(!msg, 'No msg received') + // wait 2s so the msg should have returns to the queue + await timeout(2_000) + msg = await queue.get() + t.ok(msg.id, 'Got a msg.id (sanity check)') + await queue.ack(msg.ack) + msg = await queue.get() + // no more messages + t.ok(!msg, 'No msg received') + + t.pass('Finished test ok') + t.end() }) - test('client.close()', function(t) { + test('client.close()', function (t) { t.pass('client.close()') client.close() t.end() From 3e0b48c76716a3054a08a7831488402841ec6f13 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Thu, 2 Mar 2023 11:51:30 +0000 Subject: [PATCH 12/31] =?UTF-8?q?=F0=9F=91=B7=E2=80=8D=E2=99=80=EF=B8=8F?= =?UTF-8?q?=20Bump=20GitHub=20Actions=20versions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/publish.yml | 8 ++++---- .github/workflows/test.yml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 861ca6a..8775feb 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -7,13 +7,13 @@ on: jobs: build: - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 with: - node-version: '14.x' + node-version: '18.x' registry-url: 'https://npm.pkg.github.com' - name: Install # Skip post-install to avoid malicious scripts stealing PAT diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f794d77..2fbae39 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ on: jobs: build: - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest services: mongodb: image: mongo:4.4 @@ -18,14 +18,14 @@ jobs: - 27017:27017 timeout-minutes: 10 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: # Use PAT instead of default Github token, because the default # token deliberately will not trigger another workflow run token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - - uses: actions/setup-node@v2 + - uses: actions/setup-node@v3 with: - node-version: '14.x' + node-version: '18.x' registry-url: 'https://npm.pkg.github.com' - name: Install # Skip post-install to avoid malicious scripts stealing PAT From ff780d989fbcc5ac3d38d8ead57d119cfb2add90 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Thu, 2 Mar 2023 11:49:52 +0000 Subject: [PATCH 13/31] =?UTF-8?q?=F0=9F=92=A5=20Move=20to=20Typescript?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a breaking change that moves this library to TypeScript. It's breaking because the import style has slightly changed due to the transpiler. Other than that, everything else is the same. We also: - drop our `-reedsy` version suffix, since we've now adapted this fork so heavily that it's unlikely we'll merge from upstream again - add an `.npmignore` file to avoid publishing tests and TypeScript files --- .gitignore | 4 + .npmignore | 3 + mongodb-queue.js => mongodb-queue.ts | 149 +++++++++++++++++---------- package.json | 9 +- tag.sh | 6 -- test/clean.js | 2 +- test/dead-queue.js | 2 +- test/default.js | 2 +- test/delay.js | 2 +- test/indexes.js | 2 +- test/many.js | 2 +- test/multi.js | 2 +- test/ping.js | 2 +- test/stats.js | 2 +- test/visibility.js | 2 +- tsconfig.json | 14 +++ 16 files changed, 131 insertions(+), 74 deletions(-) create mode 100644 .npmignore rename mongodb-queue.js => mongodb-queue.ts (55%) create mode 100644 tsconfig.json diff --git a/.gitignore b/.gitignore index 06ab5e5..7540984 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,7 @@ node_modules/* *.log *~ package-lock.json +*.d.ts +*.js.map +*.js +!test/*.js diff --git a/.npmignore b/.npmignore new file mode 100644 index 0000000..a09524a --- /dev/null +++ b/.npmignore @@ -0,0 +1,3 @@ +**/* +!*.d.ts +!*.js.map diff --git a/mongodb-queue.js b/mongodb-queue.ts similarity index 55% rename from mongodb-queue.js rename to mongodb-queue.ts index fb781d7..71ad175 100644 --- a/mongodb-queue.js +++ b/mongodb-queue.ts @@ -10,32 +10,74 @@ * **/ -const crypto = require('crypto') +import {randomBytes} from 'crypto'; +import {Collection, Db, Filter, FindOneAndUpdateOptions, Sort, UpdateFilter, WithId} from 'mongodb'; -function id() { - return crypto.randomBytes(16).toString('hex') +function id(): string { + return randomBytes(16).toString('hex') } -function now() { +function now(): string { return (new Date()).toISOString() } -function nowPlusSecs(secs) { +function nowPlusSecs(secs: number): string { return (new Date(Date.now() + secs * 1000)).toISOString() } -module.exports = class Queue { - constructor(db, name, opts) { +export type QueueOptions = { + visibility?: number; + delay?: number; + deadQueue?: Queue; + maxRetries?: number; +} + +export type AddOptions = { + delay?: number; +} + +export type GetOptions = { + visibility?: number; +} + +export type PingOptions = { + visibility?: number; + resetTries?: boolean; +} + +export type BaseMessage = { + payload: T; + visible: string; +} + +export type Message = BaseMessage & { + ack: string; + tries: number; + deleted?: string; +} + +export type ExternalMessage = { + id: string; + ack: string; + payload: T; + tries: number; +} + +export default class Queue { + private readonly col: Collection>>; + private readonly visibility: number; + private readonly delay: number; + private readonly maxRetries: number; + private readonly deadQueue: Queue; + + constructor(db: Db, name: string, opts: QueueOptions = {}) { if (!db) { throw new Error("mongodb-queue: provide a mongodb.MongoClient.db") } if (!name) { throw new Error("mongodb-queue: provide a queue name") } - opts = opts || {} - this.db = db - this.name = name this.col = db.collection(name) this.visibility = opts.visibility || 30 this.delay = opts.delay || 0 @@ -46,7 +88,7 @@ module.exports = class Queue { } } - async createIndexes() { + async createIndexes(): Promise { await Promise.all([ this.col.createIndex({deleted: 1, visible: 1}), this.col.createIndex({ack: 1}, {unique: true, sparse: true}), @@ -54,11 +96,11 @@ module.exports = class Queue { ]) } - async add(payload, opts = {}) { + async add(payload: T | T[], opts: AddOptions = {}): Promise { const delay = opts.delay || this.delay const visible = delay ? nowPlusSecs(delay) : now() - const msgs = [] + const msgs: BaseMessage[] = [] if (payload instanceof Array) { if (payload.length === 0) { throw new Error('Queue.add(): Array payload length must be greater than 0') @@ -81,33 +123,33 @@ module.exports = class Queue { return '' + results.insertedIds[0] } - async get(opts = {}) { + async get(opts: GetOptions = {}): Promise | null> { const visibility = opts.visibility || this.visibility - const query = { - deleted: null, + const query: Filter>> = { + deleted: {$exists: false}, visible: {$lte: now()}, } - const sort = { + const sort: Sort = { _id: 1 } - const update = { + const update: UpdateFilter> = { $inc: {tries: 1}, $set: { ack: id(), visible: nowPlusSecs(visibility), } } - const options = { + const options: FindOneAndUpdateOptions = { sort: sort, returnDocument: 'after' } const result = await this.col.findOneAndUpdate(query, update, options) - let msg = result.value - if (!msg) return + const msg = result.value as WithId>; + if (!msg) return null // convert to an external representation - msg = { + const externalMessage: ExternalMessage = { // convert '_id' to an 'id' string id: '' + msg._id, ack: msg.ack, @@ -121,32 +163,35 @@ module.exports = class Queue { // 1) add this message to the deadQueue // 2) ack this message from the regular queue // 3) call ourself to return a new message (if exists) - await this.deadQueue.add(msg) + await this.deadQueue.add(externalMessage) await this.ack(msg.ack) return this.get() } - return msg + return externalMessage } - async ping(ack, opts = {}) { + async ping(ack: string, opts: PingOptions = {}): Promise { const visibility = opts.visibility || this.visibility - const query = { + const query: Filter>> = { ack: ack, visible: {$gt: now()}, - deleted: null, + deleted: {$exists: false}, } - const update = { + const update: UpdateFilter> = { $set: { visible: nowPlusSecs(visibility) } } - const options = { + const options: FindOneAndUpdateOptions = { returnDocument: 'after' } if (opts.resetTries) { - update.$set.tries = 0 + update.$set = { + ...update.$set, + tries: 0, + } } const msg = await this.col.findOneAndUpdate(query, update, options) @@ -156,18 +201,18 @@ module.exports = class Queue { return '' + msg.value._id } - async ack(ack) { - const query = { + async ack(ack: string): Promise { + const query: Filter>> = { ack: ack, visible: {$gt: now()}, - deleted: null, + deleted: {$exists: false}, } - const update = { + const update: UpdateFilter> = { $set: { deleted: now(), } } - const options = { + const options: FindOneAndUpdateOptions = { returnDocument: 'after' } const msg = await this.col.findOneAndUpdate(query, update, options) @@ -177,42 +222,36 @@ module.exports = class Queue { return '' + msg.value._id } - async clean() { + async clean(): Promise { const query = { deleted: {$exists: true}, } - return this.col.deleteMany(query) + await this.col.deleteMany(query) } - async total() { + async total(): Promise { return this.col.countDocuments() } - async size() { - const query = { - deleted: null, + async size(): Promise { + return this.col.countDocuments({ + deleted: {$exists: false}, visible: {$lte: now()}, - } - - return this.col.countDocuments(query) + }) } - async inFlight() { - const query = { + async inFlight(): Promise { + return this.col.countDocuments({ ack: {$exists: true}, visible: {$gt: now()}, - deleted: null, - } - - return this.col.countDocuments(query) + deleted: {$exists: false}, + }) } - async done() { - const query = { + async done(): Promise { + return this.col.countDocuments({ deleted: {$exists: true}, - } - - return this.col.countDocuments(query) + }) } } diff --git a/package.json b/package.json index a0cd8a7..db139d9 100644 --- a/package.json +++ b/package.json @@ -1,15 +1,18 @@ { "name": "@reedsy/mongodb-queue", - "version": "4.0.0-reedsy-3.0.0", + "version": "5.0.0", "description": "Message queues which uses MongoDB.", "main": "mongodb-queue.js", "scripts": { + "build": "tsc --pretty", + "prepublish": "npm run build", + "pretest": "npm run build", "test": "set -e; for FILE in test/*.js; do echo --- $FILE ---; node $FILE; done" }, - "dependencies": {}, "devDependencies": { "mongodb": "^5.0.0", - "tape": "^4.10.1" + "tape": "^4.10.1", + "typescript": "^4.9.5" }, "peerDependencies": { "mongodb": "^4.0.0 || ^5.0.0" diff --git a/tag.sh b/tag.sh index eee381f..9f544d5 100755 --- a/tag.sh +++ b/tag.sh @@ -16,11 +16,5 @@ else echo "Deploying version $VERSION" fi -echo '!/dist' >> .gitignore - -git checkout -b release-$VERSION -git add .gitignore -git add --all dist/ -git commit --message "Release version $VERSION" git tag $VERSION git push origin refs/tags/$VERSION diff --git a/test/clean.js b/test/clean.js index 1624e65..5be1523 100644 --- a/test/clean.js +++ b/test/clean.js @@ -1,7 +1,7 @@ const test = require('tape') const setup = require('./setup.js') -const MongoDbQueue = require('../') +const MongoDbQueue = require('../').default setup().then(({client, db}) => { diff --git a/test/dead-queue.js b/test/dead-queue.js index 13286c4..e49097c 100644 --- a/test/dead-queue.js +++ b/test/dead-queue.js @@ -1,7 +1,7 @@ const test = require('tape') const setup = require('./setup.js') -const MongoDbQueue = require('../') +const MongoDbQueue = require('../').default setup().then(({client, db}) => { diff --git a/test/default.js b/test/default.js index 67a8d37..8d1b6ee 100644 --- a/test/default.js +++ b/test/default.js @@ -1,7 +1,7 @@ const test = require('tape') const setup = require('./setup.js') -const MongoDbQueue = require('../') +const MongoDbQueue = require('../').default setup().then(({client, db}) => { diff --git a/test/delay.js b/test/delay.js index 61a4efa..582b20d 100644 --- a/test/delay.js +++ b/test/delay.js @@ -1,7 +1,7 @@ const test = require('tape') const setup = require('./setup.js') -const MongoDbQueue = require('../') +const MongoDbQueue = require('../').default const {timeout} = require('./_timeout.js') setup().then(({client, db}) => { diff --git a/test/indexes.js b/test/indexes.js index e250b7c..fddaaa8 100644 --- a/test/indexes.js +++ b/test/indexes.js @@ -1,7 +1,7 @@ const test = require('tape') const setup = require('./setup.js') -const MongoDbQueue = require('../') +const MongoDbQueue = require('../').default setup().then(({client, db}) => { diff --git a/test/many.js b/test/many.js index 39c7659..6fc79b8 100644 --- a/test/many.js +++ b/test/many.js @@ -1,7 +1,7 @@ const test = require('tape') const setup = require('./setup.js') -const MongoDbQueue = require('../') +const MongoDbQueue = require('../').default const total = 250 diff --git a/test/multi.js b/test/multi.js index f34d6cb..9a4f0af 100644 --- a/test/multi.js +++ b/test/multi.js @@ -1,7 +1,7 @@ const test = require('tape') const setup = require('./setup.js') -const MongoDbQueue = require('../') +const MongoDbQueue = require('../').default const total = 250 diff --git a/test/ping.js b/test/ping.js index 84cb5f4..7451a37 100644 --- a/test/ping.js +++ b/test/ping.js @@ -2,7 +2,7 @@ const test = require('tape') const {timeout} = require('./_timeout') const setup = require('./setup.js') -const MongoDbQueue = require('../') +const MongoDbQueue = require('../').default setup().then(({client, db}) => { diff --git a/test/stats.js b/test/stats.js index 6679321..424984f 100644 --- a/test/stats.js +++ b/test/stats.js @@ -2,7 +2,7 @@ const test = require('tape') const {timeout} = require('./_timeout') const setup = require('./setup.js') -const MongoDbQueue = require('../') +const MongoDbQueue = require('../').default setup().then(({client, db}) => { diff --git a/test/visibility.js b/test/visibility.js index 5fa6153..2596dac 100644 --- a/test/visibility.js +++ b/test/visibility.js @@ -2,7 +2,7 @@ const test = require('tape') const {timeout} = require('./_timeout') const setup = require('./setup.js') -const MongoDbQueue = require('../') +const MongoDbQueue = require('../').default setup().then(({client, db}) => { diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..77858df --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "module": "commonjs", + "target": "es6", + "noImplicitAny": true, + "noUnusedLocals": true, + "moduleResolution": "node", + "sourceMap": true, + "declaration": true, + }, + "exclude": [ + "test/" + ], +} From d2e76c35b2c8c9c2c933d274a24e40de3f866b8c Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Thu, 2 Mar 2023 13:12:50 +0000 Subject: [PATCH 14/31] =?UTF-8?q?=F0=9F=91=B7=E2=80=8D=E2=99=80=EF=B8=8F?= =?UTF-8?q?=20Fix=20release?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `npm prepublish` was deprecated. This change updates the release to use `npm prepare` instead --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index db139d9..ffadc43 100644 --- a/package.json +++ b/package.json @@ -1,11 +1,11 @@ { "name": "@reedsy/mongodb-queue", - "version": "5.0.0", + "version": "5.0.1", "description": "Message queues which uses MongoDB.", "main": "mongodb-queue.js", "scripts": { "build": "tsc --pretty", - "prepublish": "npm run build", + "prepare": "npm run build", "pretest": "npm run build", "test": "set -e; for FILE in test/*.js; do echo --- $FILE ---; node $FILE; done" }, From e2045fbaabc47cf8531a0c030be7eca72fc16245 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Thu, 2 Mar 2023 13:22:11 +0000 Subject: [PATCH 15/31] =?UTF-8?q?=F0=9F=9A=A8=20Add=20linting?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .eslintignore | 8 ++ .eslintrc.yml | 6 + .github/workflows/test.yml | 2 + .npmrc | 1 + mongodb-queue.ts | 170 +++++++++++++-------------- package.json | 3 + test/_timeout.js | 4 +- test/clean.js | 78 ++++++------- test/dead-queue.js | 156 ++++++++++++------------- test/default.js | 110 +++++++++--------- test/delay.js | 110 +++++++++--------- test/indexes.js | 32 +++-- test/many.js | 95 ++++++++------- test/multi.js | 70 ++++++----- test/ping.js | 231 ++++++++++++++++++------------------- test/setup.js | 22 ++-- test/stats.js | 120 +++++++++---------- test/visibility.js | 177 ++++++++++++++-------------- 18 files changed, 689 insertions(+), 706 deletions(-) create mode 100644 .eslintignore create mode 100644 .eslintrc.yml create mode 100644 .npmrc diff --git a/.eslintignore b/.eslintignore new file mode 100644 index 0000000..47beb92 --- /dev/null +++ b/.eslintignore @@ -0,0 +1,8 @@ +*.json +*.sh +*.yml +*.md +*.d.ts +*.js.map +*.js +!test/*.js diff --git a/.eslintrc.yml b/.eslintrc.yml new file mode 100644 index 0000000..08f1c63 --- /dev/null +++ b/.eslintrc.yml @@ -0,0 +1,6 @@ +extends: + - 'plugin:@reedsy/recommended' +parserOptions: + project: './tsconfig.json' +env: + node: true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2fbae39..6e1cab4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -36,6 +36,8 @@ jobs: NODE_AUTH_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - name: Post-install run: npm rebuild && npm run prepare --if-present + - name: Lint + run: npm run lint - name: Test run: npm test - name: Tag diff --git a/.npmrc b/.npmrc new file mode 100644 index 0000000..f06c805 --- /dev/null +++ b/.npmrc @@ -0,0 +1 @@ +@reedsy:registry=https://npm.pkg.github.com diff --git a/mongodb-queue.ts b/mongodb-queue.ts index 71ad175..82db763 100644 --- a/mongodb-queue.ts +++ b/mongodb-queue.ts @@ -14,15 +14,15 @@ import {randomBytes} from 'crypto'; import {Collection, Db, Filter, FindOneAndUpdateOptions, Sort, UpdateFilter, WithId} from 'mongodb'; function id(): string { - return randomBytes(16).toString('hex') + return randomBytes(16).toString('hex'); } function now(): string { - return (new Date()).toISOString() + return (new Date()).toISOString(); } function nowPlusSecs(secs: number): string { - return (new Date(Date.now() + secs * 1000)).toISOString() + return (new Date(Date.now() + secs * 1000)).toISOString(); } export type QueueOptions = { @@ -30,123 +30,123 @@ export type QueueOptions = { delay?: number; deadQueue?: Queue; maxRetries?: number; -} +}; export type AddOptions = { delay?: number; -} +}; export type GetOptions = { visibility?: number; -} +}; export type PingOptions = { visibility?: number; resetTries?: boolean; -} +}; -export type BaseMessage = { +export type BaseMessage = { payload: T; visible: string; -} +}; -export type Message = BaseMessage & { +export type Message = BaseMessage & { ack: string; tries: number; deleted?: string; -} +}; -export type ExternalMessage = { +export type ExternalMessage = { id: string; ack: string; payload: T; tries: number; -} +}; -export default class Queue { +export default class Queue { private readonly col: Collection>>; private readonly visibility: number; private readonly delay: number; private readonly maxRetries: number; private readonly deadQueue: Queue; - constructor(db: Db, name: string, opts: QueueOptions = {}) { + public constructor(db: Db, name: string, opts: QueueOptions = {}) { if (!db) { - throw new Error("mongodb-queue: provide a mongodb.MongoClient.db") + throw new Error('mongodb-queue: provide a mongodb.MongoClient.db'); } if (!name) { - throw new Error("mongodb-queue: provide a queue name") + throw new Error('mongodb-queue: provide a queue name'); } - this.col = db.collection(name) - this.visibility = opts.visibility || 30 - this.delay = opts.delay || 0 + this.col = db.collection(name); + this.visibility = opts.visibility || 30; + this.delay = opts.delay || 0; if (opts.deadQueue) { - this.deadQueue = opts.deadQueue - this.maxRetries = opts.maxRetries || 5 + this.deadQueue = opts.deadQueue; + this.maxRetries = opts.maxRetries || 5; } } - async createIndexes(): Promise { + public async createIndexes(): Promise { await Promise.all([ this.col.createIndex({deleted: 1, visible: 1}), this.col.createIndex({ack: 1}, {unique: true, sparse: true}), - this.col.createIndex({deleted: 1}, {sparse: true}) - ]) + this.col.createIndex({deleted: 1}, {sparse: true}), + ]); } - async add(payload: T | T[], opts: AddOptions = {}): Promise { - const delay = opts.delay || this.delay - const visible = delay ? nowPlusSecs(delay) : now() + public async add(payload: T | T[], opts: AddOptions = {}): Promise { + const delay = opts.delay || this.delay; + const visible = delay ? nowPlusSecs(delay) : now(); - const msgs: BaseMessage[] = [] + const msgs: BaseMessage[] = []; if (payload instanceof Array) { if (payload.length === 0) { - throw new Error('Queue.add(): Array payload length must be greater than 0') + throw new Error('Queue.add(): Array payload length must be greater than 0'); } - payload.forEach(function (payload) { + payload.forEach(function(payload) { msgs.push({ visible: visible, payload: payload, - }) - }) + }); + }); } else { msgs.push({ visible: visible, payload: payload, - }) + }); } - const results = await this.col.insertMany(msgs) - if (payload instanceof Array) return '' + results.insertedIds - return '' + results.insertedIds[0] + const results = await this.col.insertMany(msgs); + if (payload instanceof Array) return '' + results.insertedIds; + return '' + results.insertedIds[0]; } - async get(opts: GetOptions = {}): Promise | null> { - const visibility = opts.visibility || this.visibility + public async get(opts: GetOptions = {}): Promise | null> { + const visibility = opts.visibility || this.visibility; const query: Filter>> = { deleted: {$exists: false}, visible: {$lte: now()}, - } + }; const sort: Sort = { - _id: 1 - } + _id: 1, + }; const update: UpdateFilter> = { $inc: {tries: 1}, $set: { ack: id(), visible: nowPlusSecs(visibility), - } - } + }, + }; const options: FindOneAndUpdateOptions = { sort: sort, - returnDocument: 'after' - } + returnDocument: 'after', + }; - const result = await this.col.findOneAndUpdate(query, update, options) + const result = await this.col.findOneAndUpdate(query, update, options); const msg = result.value as WithId>; - if (!msg) return null + if (!msg) return null; // convert to an external representation const externalMessage: ExternalMessage = { @@ -155,7 +155,7 @@ export default class Queue { ack: msg.ack, payload: msg.payload, tries: msg.tries, - } + }; // check the tries if (this.deadQueue && msg.tries > this.maxRetries) { @@ -163,95 +163,95 @@ export default class Queue { // 1) add this message to the deadQueue // 2) ack this message from the regular queue // 3) call ourself to return a new message (if exists) - await this.deadQueue.add(externalMessage) - await this.ack(msg.ack) - return this.get() + await this.deadQueue.add(externalMessage); + await this.ack(msg.ack); + return this.get(); } - return externalMessage + return externalMessage; } - async ping(ack: string, opts: PingOptions = {}): Promise { - const visibility = opts.visibility || this.visibility + public async ping(ack: string, opts: PingOptions = {}): Promise { + const visibility = opts.visibility || this.visibility; const query: Filter>> = { ack: ack, visible: {$gt: now()}, deleted: {$exists: false}, - } + }; const update: UpdateFilter> = { $set: { - visible: nowPlusSecs(visibility) - } - } + visible: nowPlusSecs(visibility), + }, + }; const options: FindOneAndUpdateOptions = { - returnDocument: 'after' - } + returnDocument: 'after', + }; if (opts.resetTries) { update.$set = { ...update.$set, tries: 0, - } + }; } - const msg = await this.col.findOneAndUpdate(query, update, options) + const msg = await this.col.findOneAndUpdate(query, update, options); if (!msg.value) { - throw new Error("Queue.ping(): Unidentified ack : " + ack) + throw new Error('Queue.ping(): Unidentified ack : ' + ack); } - return '' + msg.value._id + return '' + msg.value._id; } - async ack(ack: string): Promise { + public async ack(ack: string): Promise { const query: Filter>> = { ack: ack, visible: {$gt: now()}, deleted: {$exists: false}, - } + }; const update: UpdateFilter> = { $set: { deleted: now(), - } - } + }, + }; const options: FindOneAndUpdateOptions = { - returnDocument: 'after' - } - const msg = await this.col.findOneAndUpdate(query, update, options) + returnDocument: 'after', + }; + const msg = await this.col.findOneAndUpdate(query, update, options); if (!msg.value) { - throw new Error("Queue.ack(): Unidentified ack : " + ack) + throw new Error('Queue.ack(): Unidentified ack : ' + ack); } - return '' + msg.value._id + return '' + msg.value._id; } - async clean(): Promise { + public async clean(): Promise { const query = { deleted: {$exists: true}, - } + }; - await this.col.deleteMany(query) + await this.col.deleteMany(query); } - async total(): Promise { - return this.col.countDocuments() + public async total(): Promise { + return this.col.countDocuments(); } - async size(): Promise { + public async size(): Promise { return this.col.countDocuments({ deleted: {$exists: false}, visible: {$lte: now()}, - }) + }); } - async inFlight(): Promise { + public async inFlight(): Promise { return this.col.countDocuments({ ack: {$exists: true}, visible: {$gt: now()}, deleted: {$exists: false}, - }) + }); } - async done(): Promise { + public async done(): Promise { return this.col.countDocuments({ deleted: {$exists: true}, - }) + }); } } diff --git a/package.json b/package.json index ffadc43..da6ab37 100644 --- a/package.json +++ b/package.json @@ -5,11 +5,14 @@ "main": "mongodb-queue.js", "scripts": { "build": "tsc --pretty", + "lint": "eslint '**/*' --max-warnings 0", "prepare": "npm run build", "pretest": "npm run build", "test": "set -e; for FILE in test/*.js; do echo --- $FILE ---; node $FILE; done" }, "devDependencies": { + "@reedsy/eslint-plugin": "^0.14.2", + "eslint": "^8.35.0", "mongodb": "^5.0.0", "tape": "^4.10.1", "typescript": "^4.9.5" diff --git a/test/_timeout.js b/test/_timeout.js index 2ba20bc..c87a13e 100644 --- a/test/_timeout.js +++ b/test/_timeout.js @@ -1,7 +1,7 @@ function timeout(millis) { - return new Promise((resolve) => setTimeout(resolve, millis)) + return new Promise((resolve) => setTimeout(resolve, millis)); } module.exports = { timeout, -} +}; diff --git a/test/clean.js b/test/clean.js index 5be1523..21d64a1 100644 --- a/test/clean.js +++ b/test/clean.js @@ -1,47 +1,43 @@ -const test = require('tape') +const test = require('tape'); -const setup = require('./setup.js') -const MongoDbQueue = require('../').default +const setup = require('./setup.js'); +const MongoDbQueue = require('../').default; setup().then(({client, db}) => { + test('clean: check deleted messages are deleted', async function(t) { + const q = new MongoDbQueue(db, 'clean', {visibility: 3}); - test('clean: check deleted messages are deleted', async function (t) { - const q = new MongoDbQueue(db, 'clean', {visibility: 3}) - let msg - let id + t.equal(await q.size(), 0, 'There is currently nothing on the queue'); + t.equal(await q.total(), 0, 'There is currently nothing in the queue at all'); + await q.clean(); + t.equal(await q.size(), 0, 'There is currently nothing on the queue'); + t.equal(await q.total(), 0, 'There is currently nothing in the queue at all'); + await q.add('Hello, World!'); + await q.clean(); + t.equal(await q.size(), 1, 'Queue size is correct'); + t.equal(await q.total(), 1, 'Queue total is correct'); + const msg = await q.get(); + t.ok(msg.id, 'Got a msg.id (sanity check)'); + t.equal(await q.size(), 0, 'Queue size is correct'); + t.equal(await q.total(), 1, 'Queue total is correct'); + await q.clean(); + t.equal(await q.size(), 0, 'Queue size is correct'); + t.equal(await q.total(), 1, 'Queue total is correct'); + const id = await q.ack(msg.ack); + t.ok(id, 'Received an id when acking this message'); + t.equal(await q.size(), 0, 'Queue size is correct'); + t.equal(await q.total(), 1, 'Queue total is correct'); + await q.clean(); + t.equal(await q.size(), 0, 'Queue size is correct'); + t.equal(await q.total(), 0, 'Queue total is correct'); - t.equal(await q.size(), 0, 'There is currently nothing on the queue') - t.equal(await q.total(), 0, 'There is currently nothing in the queue at all') - await q.clean() - t.equal(await q.size(), 0, 'There is currently nothing on the queue') - t.equal(await q.total(), 0, 'There is currently nothing in the queue at all') - await q.add('Hello, World!') - await q.clean() - t.equal(await q.size(), 1, 'Queue size is correct') - t.equal(await q.total(), 1, 'Queue total is correct') - msg = await q.get() - t.ok(msg.id, 'Got a msg.id (sanity check)') - t.equal(await q.size(), 0, 'Queue size is correct') - t.equal(await q.total(), 1, 'Queue total is correct') - await q.clean() - t.equal(await q.size(), 0, 'Queue size is correct') - t.equal(await q.total(), 1, 'Queue total is correct') - id = await q.ack(msg.ack) - t.ok(id, 'Received an id when acking this message') - t.equal(await q.size(), 0, 'Queue size is correct') - t.equal(await q.total(), 1, 'Queue total is correct') - await q.clean() - t.equal(await q.size(), 0, 'Queue size is correct') - t.equal(await q.total(), 0, 'Queue total is correct') + t.pass('Finished test ok'); + t.end(); + }); - t.pass('Finished test ok') - t.end() - }) - - test('client.close()', function(t) { - t.pass('client.close()') - client.close() - t.end() - }) - -}) + test('client.close()', function(t) { + t.pass('client.close()'); + client.close(); + t.end(); + }); +}); diff --git a/test/dead-queue.js b/test/dead-queue.js index e49097c..d32a1eb 100644 --- a/test/dead-queue.js +++ b/test/dead-queue.js @@ -1,83 +1,79 @@ -const test = require('tape') +const test = require('tape'); -const setup = require('./setup.js') -const MongoDbQueue = require('../').default +const setup = require('./setup.js'); +const MongoDbQueue = require('../').default; setup().then(({client, db}) => { - - test('first test', function (t) { - const queue = new MongoDbQueue(db, 'queue', {visibility: 3, deadQueue: 'dead-queue'}) - t.ok(queue, 'Queue created ok') - t.end() - }); - - test('single message going over 5 tries, should appear on dead-queue', async function (t) { - const deadQueue = new MongoDbQueue(db, 'dead-queue') - const queue = new MongoDbQueue(db, 'queue', {visibility: 1, deadQueue: deadQueue}) - let msg - let origId - - origId = await queue.add('Hello, World!') - t.ok(origId, 'Received an id for this message') - - await queue.get() - - for (let i = 1; i <= 5; i++) { - await queue.get() - await new Promise((resolve) => setTimeout(function () { - t.pass(`Expiration #${i}`) - resolve() - }, 2 * 1000)) - } - - msg = await queue.get() - t.ok(!msg, 'No msg received') - - msg = await deadQueue.get() - t.ok(msg.id, 'Got a message id from the deadQueue') - t.equal(msg.payload.id, origId, 'Got the same message id as the original message') - t.equal(msg.payload.payload, 'Hello, World!', 'Got the same as the original message') - t.equal(msg.payload.tries, 6, 'Got the tries as 6') - - t.end() - }) - - test('two messages, with first going over 3 tries', async function (t) { - const deadQueue = new MongoDbQueue(db, 'dead-queue-2') - const queue = new MongoDbQueue(db, 'queue-2', {visibility: 1, deadQueue: deadQueue, maxRetries: 3}) - let msg - let origId, origId2 - - origId = await queue.add('Hello, World!') - t.ok(origId, 'Received an id for this message') - origId2 = await queue.add('Part II') - t.ok(origId2, 'Received an id for this message') - - for (let i = 1; i <= 3; i++) { - msg = await queue.get() - t.equal(msg.id, origId, 'We return the first message on first go') - await new Promise((resolve) => setTimeout(function () { - t.pass(`Expiration #${i}`) - resolve() - }, 2 * 1000)) - } - - msg = await queue.get() - t.equal(msg.id, origId2, 'Got the ID of the 2nd message') - t.equal(msg.payload, 'Part II', 'Got the same payload as the 2nd message') - - msg = await deadQueue.get() - t.ok(msg.id, 'Got a message id from the deadQueue') - t.equal(msg.payload.id, origId, 'Got the same message id as the original message') - t.equal(msg.payload.payload, 'Hello, World!', 'Got the same as the original message') - t.equal(msg.payload.tries, 4, 'Got the tries as 4') - t.end() - }) - - test('client.close()', function (t) { - t.pass('client.close()') - client.close() - t.end() - }) - -}) + test('first test', function(t) { + const queue = new MongoDbQueue(db, 'queue', {visibility: 3, deadQueue: 'dead-queue'}); + t.ok(queue, 'Queue created ok'); + t.end(); + }); + + test('single message going over 5 tries, should appear on dead-queue', async function(t) { + const deadQueue = new MongoDbQueue(db, 'dead-queue'); + const queue = new MongoDbQueue(db, 'queue', {visibility: 1, deadQueue: deadQueue}); + let msg; + + const origId = await queue.add('Hello, World!'); + t.ok(origId, 'Received an id for this message'); + + await queue.get(); + + for (let i = 1; i <= 5; i++) { + await queue.get(); + await new Promise((resolve) => setTimeout(function() { + t.pass(`Expiration #${i}`); + resolve(); + }, 2 * 1000)); + } + + msg = await queue.get(); + t.ok(!msg, 'No msg received'); + + msg = await deadQueue.get(); + t.ok(msg.id, 'Got a message id from the deadQueue'); + t.equal(msg.payload.id, origId, 'Got the same message id as the original message'); + t.equal(msg.payload.payload, 'Hello, World!', 'Got the same as the original message'); + t.equal(msg.payload.tries, 6, 'Got the tries as 6'); + + t.end(); + }); + + test('two messages, with first going over 3 tries', async function(t) { + const deadQueue = new MongoDbQueue(db, 'dead-queue-2'); + const queue = new MongoDbQueue(db, 'queue-2', {visibility: 1, deadQueue: deadQueue, maxRetries: 3}); + let msg; + + const origId = await queue.add('Hello, World!'); + t.ok(origId, 'Received an id for this message'); + const origId2 = await queue.add('Part II'); + t.ok(origId2, 'Received an id for this message'); + + for (let i = 1; i <= 3; i++) { + msg = await queue.get(); + t.equal(msg.id, origId, 'We return the first message on first go'); + await new Promise((resolve) => setTimeout(function() { + t.pass(`Expiration #${i}`); + resolve(); + }, 2 * 1000)); + } + + msg = await queue.get(); + t.equal(msg.id, origId2, 'Got the ID of the 2nd message'); + t.equal(msg.payload, 'Part II', 'Got the same payload as the 2nd message'); + + msg = await deadQueue.get(); + t.ok(msg.id, 'Got a message id from the deadQueue'); + t.equal(msg.payload.id, origId, 'Got the same message id as the original message'); + t.equal(msg.payload.payload, 'Hello, World!', 'Got the same as the original message'); + t.equal(msg.payload.tries, 4, 'Got the tries as 4'); + t.end(); + }); + + test('client.close()', function(t) { + t.pass('client.close()'); + client.close(); + t.end(); + }); +}); diff --git a/test/default.js b/test/default.js index 8d1b6ee..9af5281 100644 --- a/test/default.js +++ b/test/default.js @@ -1,68 +1,64 @@ -const test = require('tape') +const test = require('tape'); -const setup = require('./setup.js') -const MongoDbQueue = require('../').default +const setup = require('./setup.js'); +const MongoDbQueue = require('../').default; setup().then(({client, db}) => { + test('first test', function(t) { + const queue = new MongoDbQueue(db, 'default'); + t.ok(queue, 'Queue created ok'); + t.end(); + }); - test('first test', function (t) { - const queue = new MongoDbQueue(db, 'default') - t.ok(queue, 'Queue created ok') - t.end() - }); + test('single round trip', async function(t) { + const queue = new MongoDbQueue(db, 'default'); + let id; - test('single round trip', async function (t) { - const queue = new MongoDbQueue(db, 'default') - let msg - let id + id = await queue.add('Hello, World!'); + t.ok(id, 'Received an id for this message'); - id = await queue.add('Hello, World!') - t.ok(id, 'Received an id for this message') + const msg = await queue.get(); + t.ok(msg.id, 'Got a msg.id'); + t.equal(typeof msg.id, 'string', 'msg.id is a string'); + t.ok(msg.ack, 'Got a msg.ack'); + t.equal(typeof msg.ack, 'string', 'msg.ack is a string'); + t.ok(msg.tries, 'Got a msg.tries'); + t.equal(typeof msg.tries, 'number', 'msg.tries is a number'); + t.equal(msg.tries, 1, 'msg.tries is currently one'); + t.equal(msg.payload, 'Hello, World!', 'Payload is correct'); - msg = await queue.get() - t.ok(msg.id, 'Got a msg.id') - t.equal(typeof msg.id, 'string', 'msg.id is a string') - t.ok(msg.ack, 'Got a msg.ack') - t.equal(typeof msg.ack, 'string', 'msg.ack is a string') - t.ok(msg.tries, 'Got a msg.tries') - t.equal(typeof msg.tries, 'number', 'msg.tries is a number') - t.equal(msg.tries, 1, 'msg.tries is currently one') - t.equal(msg.payload, 'Hello, World!', 'Payload is correct') + id = await queue.ack(msg.ack); + t.ok(id, 'Received an id when acking this message'); + t.end(); + }); - id = await queue.ack(msg.ack) - t.ok(id, 'Received an id when acking this message') - t.end() - }) + test("single round trip, can't be acked again", async function(t) { + const queue = new MongoDbQueue(db, 'default'); + let id; - test("single round trip, can't be acked again", async function (t) { - const queue = new MongoDbQueue(db, 'default') - let msg - let id + id = await queue.add('Hello, World!'); + t.ok(id, 'Received an id for this message'); + const msg = await queue.get(); + t.ok(msg.id, 'Got a msg.id'); + t.equal(typeof msg.id, 'string', 'msg.id is a string'); + t.ok(msg.ack, 'Got a msg.ack'); + t.equal(typeof msg.ack, 'string', 'msg.ack is a string'); + t.ok(msg.tries, 'Got a msg.tries'); + t.equal(typeof msg.tries, 'number', 'msg.tries is a number'); + t.equal(msg.tries, 1, 'msg.tries is currently one'); + t.equal(msg.payload, 'Hello, World!', 'Payload is correct'); + id = await queue.ack(msg.ack); + t.ok(id, 'Received an id when acking this message'); + id = await queue.ack(msg.ack) + .catch((err) => t.ok(err, 'There is an error when acking the message again')); - id = await queue.add('Hello, World!') - t.ok(id, 'Received an id for this message') - msg = await queue.get() - t.ok(msg.id, 'Got a msg.id') - t.equal(typeof msg.id, 'string', 'msg.id is a string') - t.ok(msg.ack, 'Got a msg.ack') - t.equal(typeof msg.ack, 'string', 'msg.ack is a string') - t.ok(msg.tries, 'Got a msg.tries') - t.equal(typeof msg.tries, 'number', 'msg.tries is a number') - t.equal(msg.tries, 1, 'msg.tries is currently one') - t.equal(msg.payload, 'Hello, World!', 'Payload is correct') - id = await queue.ack(msg.ack) - t.ok(id, 'Received an id when acking this message') - id = await queue.ack(msg.ack) - .catch((err) => t.ok(err, 'There is an error when acking the message again')) + t.ok(!id, 'No id received when trying to ack an already deleted message'); + t.end(); + }); - t.ok(!id, 'No id received when trying to ack an already deleted message') - t.end() - }) - - test('client.close()', function (t) { - t.pass('client.close()') - client.close() - t.end() - }) - -}) + test('client.close()', function(t) { + t.pass('client.close()'); + client.close(); + t.end(); + }); +}); diff --git a/test/delay.js b/test/delay.js index 582b20d..0dc7af6 100644 --- a/test/delay.js +++ b/test/delay.js @@ -1,60 +1,56 @@ -const test = require('tape') +const test = require('tape'); -const setup = require('./setup.js') -const MongoDbQueue = require('../').default -const {timeout} = require('./_timeout.js') +const setup = require('./setup.js'); +const MongoDbQueue = require('../').default; +const {timeout} = require('./_timeout.js'); setup().then(({client, db}) => { - - test('delay: check messages on this queue are returned after the delay', async function (t) { - const queue = new MongoDbQueue(db, 'delay', {delay: 3}) - let id - let msg - - id = await queue.add('Hello, World!') - t.ok(id, 'There is an id returned when adding a message.') - // get something now and it shouldn't be there - msg = await queue.get() - t.ok(!msg, 'No msg received') - await timeout(4_000) - // get something now and it SHOULD be there - msg = await queue.get() - t.ok(msg.id, 'Got a message id now that the message delay has passed') - await queue.ack(msg.ack) - msg = await queue.get() - // no more messages - t.ok(!msg, 'No more messages') - t.pass('Finished test ok') - t.end() - }) - - test('delay: check an individual message delay overrides the queue delay', async function (t) { - const queue = new MongoDbQueue(db, 'delay') - let id - let msg - - id = await queue.add('I am delayed by 3 seconds', {delay: 3}) - t.ok(id, 'There is an id returned when adding a message.') - // get something now and it shouldn't be there - msg = await queue.get() - t.ok(!msg, 'No msg received') - await timeout(4_000) - // get something now and it SHOULD be there - msg = await queue.get() - t.ok(msg.id, 'Got a message id now that the message delay has passed') - await queue.ack(msg.ack) - msg = await queue.get() - // no more messages - t.ok(!msg, 'No more messages') - - t.pass('Finished test ok') - t.end() - }) - - test('client.close()', function (t) { - t.pass('client.close()') - client.close() - t.end() - }) - -}) + test('delay: check messages on this queue are returned after the delay', async function(t) { + const queue = new MongoDbQueue(db, 'delay', {delay: 3}); + let msg; + + const id = await queue.add('Hello, World!'); + t.ok(id, 'There is an id returned when adding a message.'); + // get something now and it shouldn't be there + msg = await queue.get(); + t.ok(!msg, 'No msg received'); + await timeout(4000); + // get something now and it SHOULD be there + msg = await queue.get(); + t.ok(msg.id, 'Got a message id now that the message delay has passed'); + await queue.ack(msg.ack); + msg = await queue.get(); + // no more messages + t.ok(!msg, 'No more messages'); + t.pass('Finished test ok'); + t.end(); + }); + + test('delay: check an individual message delay overrides the queue delay', async function(t) { + const queue = new MongoDbQueue(db, 'delay'); + let msg; + + const id = await queue.add('I am delayed by 3 seconds', {delay: 3}); + t.ok(id, 'There is an id returned when adding a message.'); + // get something now and it shouldn't be there + msg = await queue.get(); + t.ok(!msg, 'No msg received'); + await timeout(4000); + // get something now and it SHOULD be there + msg = await queue.get(); + t.ok(msg.id, 'Got a message id now that the message delay has passed'); + await queue.ack(msg.ack); + msg = await queue.get(); + // no more messages + t.ok(!msg, 'No more messages'); + + t.pass('Finished test ok'); + t.end(); + }); + + test('client.close()', function(t) { + t.pass('client.close()'); + client.close(); + t.end(); + }); +}); diff --git a/test/indexes.js b/test/indexes.js index fddaaa8..791932f 100644 --- a/test/indexes.js +++ b/test/indexes.js @@ -1,22 +1,20 @@ -const test = require('tape') +const test = require('tape'); -const setup = require('./setup.js') -const MongoDbQueue = require('../').default +const setup = require('./setup.js'); +const MongoDbQueue = require('../').default; setup().then(({client, db}) => { + test('visibility: check message is back in queue after 3s', async function(t) { + const queue = new MongoDbQueue(db, 'visibility', {visibility: 3}); - test('visibility: check message is back in queue after 3s', async function(t) { - const queue = new MongoDbQueue(db, 'visibility', { visibility : 3 }) + await queue.createIndexes(); + t.pass('Indexes created'); + t.end(); + }); - await queue.createIndexes() - t.pass('Indexes created') - t.end() - }) - - test('client.close()', function(t) { - t.pass('client.close()') - client.close() - t.end() - }) - -}) + test('client.close()', function(t) { + t.pass('client.close()'); + client.close(); + t.end(); + }); +}); diff --git a/test/many.js b/test/many.js index 6fc79b8..b52fa2f 100644 --- a/test/many.js +++ b/test/many.js @@ -1,53 +1,50 @@ -const test = require('tape') +const test = require('tape'); -const setup = require('./setup.js') -const MongoDbQueue = require('../').default +const setup = require('./setup.js'); +const MongoDbQueue = require('../').default; -const total = 250 +const total = 250; setup().then(({client, db}) => { - - test('many: add ' + total + ' messages, get ' + total + ' back', async function (t) { - const queue = new MongoDbQueue(db, 'many') - const msgs = [] - const msgsToQueue = [] - - - for (let i = 0; i < total; i++) { - msgsToQueue.push('no=' + i) - } - await queue.add(msgsToQueue) - t.pass('All ' + total + ' messages sent to MongoDB') - - async function getOne() { - const msg = await queue.get() - if (!msg) return t.fail('Failed getting a message') - msgs.push(msg) - if (msgs.length !== total) return getOne() - t.pass('Received all ' + total + ' messages') - } - await getOne() - - await Promise.all( - msgs.map((msg) => queue.ack(msg.ack)) - ) - - t.pass('Acked all ' + total + ' messages') - t.pass('Finished test ok') - t.end() - }) - - test('many: add no messages, receive err in callback', async function (t) { - const queue = new MongoDbQueue(db, 'many') - await queue.add([]) - .catch(() => t.pass('got error')) - t.end() - }) - - test('client.close()', function (t) { - t.pass('client.close()') - client.close() - t.end() - }) - -}) + test('many: add ' + total + ' messages, get ' + total + ' back', async function(t) { + const queue = new MongoDbQueue(db, 'many'); + const msgs = []; + const msgsToQueue = []; + + for (let i = 0; i < total; i++) { + msgsToQueue.push('no=' + i); + } + await queue.add(msgsToQueue); + t.pass('All ' + total + ' messages sent to MongoDB'); + + async function getOne() { + const msg = await queue.get(); + if (!msg) return t.fail('Failed getting a message'); + msgs.push(msg); + if (msgs.length !== total) return getOne(); + t.pass('Received all ' + total + ' messages'); + } + await getOne(); + + await Promise.all( + msgs.map((msg) => queue.ack(msg.ack)), + ); + + t.pass('Acked all ' + total + ' messages'); + t.pass('Finished test ok'); + t.end(); + }); + + test('many: add no messages, receive err in callback', async function(t) { + const queue = new MongoDbQueue(db, 'many'); + await queue.add([]) + .catch(() => t.pass('got error')); + t.end(); + }); + + test('client.close()', function(t) { + t.pass('client.close()'); + client.close(); + t.end(); + }); +}); diff --git a/test/multi.js b/test/multi.js index 9a4f0af..8508a85 100644 --- a/test/multi.js +++ b/test/multi.js @@ -1,40 +1,38 @@ -const test = require('tape') +const test = require('tape'); -const setup = require('./setup.js') -const MongoDbQueue = require('../').default +const setup = require('./setup.js'); +const MongoDbQueue = require('../').default; -const total = 250 +const total = 250; setup().then(({client, db}) => { - - test('multi: add ' + total + ' messages, get ' + total + ' back', async function (t) { - const queue = new MongoDbQueue(db, 'multi') - const msgs = [] - - for (let i = 0; i < total; i++) await queue.add('no=' + i) - t.pass('All ' + total + ' messages sent to MongoDB') - - async function getOne() { - const msg = await queue.get() - msgs.push(msg) - if (msgs.length !== total) return getOne() - t.pass('Received all ' + total + ' messages') - } - await getOne() - - await Promise.all( - msgs.map((msg) => queue.ack(msg.ack)) - ) - - t.pass('Acked all ' + total + ' messages') - - t.end() - }) - - test('client.close()', function (t) { - t.pass('client.close()') - client.close() - t.end() - }) - -}) + test('multi: add ' + total + ' messages, get ' + total + ' back', async function(t) { + const queue = new MongoDbQueue(db, 'multi'); + const msgs = []; + + for (let i = 0; i < total; i++) await queue.add('no=' + i); + t.pass('All ' + total + ' messages sent to MongoDB'); + + async function getOne() { + const msg = await queue.get(); + msgs.push(msg); + if (msgs.length !== total) return getOne(); + t.pass('Received all ' + total + ' messages'); + } + await getOne(); + + await Promise.all( + msgs.map((msg) => queue.ack(msg.ack)), + ); + + t.pass('Acked all ' + total + ' messages'); + + t.end(); + }); + + test('client.close()', function(t) { + t.pass('client.close()'); + client.close(); + t.end(); + }); +}); diff --git a/test/ping.js b/test/ping.js index 7451a37..5c5a5d1 100644 --- a/test/ping.js +++ b/test/ping.js @@ -1,120 +1,117 @@ -const test = require('tape') -const {timeout} = require('./_timeout') +const test = require('tape'); +const {timeout} = require('./_timeout'); -const setup = require('./setup.js') -const MongoDbQueue = require('../').default +const setup = require('./setup.js'); +const MongoDbQueue = require('../').default; setup().then(({client, db}) => { - - test('ping: check a retrieved message with a ping can still be acked', async function (t) { - const queue = new MongoDbQueue(db, 'ping', {visibility: 5}) - let msg - let id - - id = await queue.add('Hello, World!') - t.ok(id, 'There is an id returned when adding a message.') - // get something now and it shouldn't be there - msg = await queue.get() - t.ok(msg.id, 'Got this message id') - await timeout(4_000) - // ping this message so it will be kept alive longer, another 5s - id = await queue.ping(msg.ack) - t.ok(id, 'Received an id when acking this message') - await timeout(4_000) - id = await queue.ack(msg.ack) - t.ok(id, 'Received an id when acking this message') - msg = await queue.get() - t.ok(!msg, 'No message when getting from an empty queue') - - t.pass('Finished test ok') - t.end() - }) - - test("ping: check that an acked message can't be pinged", async function (t) { - const queue = new MongoDbQueue(db, 'ping', {visibility: 5}) - let msg - let id - - id = await queue.add('Hello, World!') - t.ok(id, 'There is an id returned when adding a message.') - // get something now and it shouldn't be there - msg = await queue.get() - t.ok(msg.id, 'Got this message id') - // ack the message - id = await queue.ack(msg.ack) - t.ok(id, 'Received an id when acking this message') - // ping this message, even though it has been acked - id = await queue.ping(msg.ack) - .catch((err) => t.ok(err, 'Error when pinging an acked message')) - t.ok(!id, 'Received no id when pinging an acked message') - - t.pass('Finished test ok') - t.end() - }) - - test("ping: check visibility option overrides the queue visibility", async function (t) { - const queue = new MongoDbQueue(db, 'ping', {visibility: 3}) - let msg - let id - - id = await queue.add('Hello, World!') - t.ok(id, 'There is an id returned when adding a message.') - msg = await queue.get() - // message should reset in three seconds - t.ok(msg.id, 'Got a msg.id (sanity check)') - await timeout(2_000) - // ping this message so it will be kept alive longer, another 5s instead of 3s - id = await queue.ping(msg.ack, {visibility: 5}) - t.ok(id, 'Received an id when acking this message') - // wait 4s so the msg would normally have returns to the queue - await timeout(4_000) - msg = await queue.get() - // messages should not be back yet - t.ok(!msg, 'No msg received') - // wait 2s so the msg should have returns to the queue - await timeout(2_000) - msg = await queue.get() - // yes, there should be a message on the queue again - t.ok(msg.id, 'Got a msg.id (sanity check)') - await queue.ack(msg.ack) - msg = await queue.get() - // no more messages - t.ok(!msg, 'No msg received') - - t.pass('Finished test ok') - t.end() - }) - - test("ping: reset tries", async function (t) { - const queue = new MongoDbQueue(db, 'ping', {visibility: 3}) - let msg - let id - - id = await queue.add('Hello, World!') - t.ok(id, 'There is an id returned when adding a message.') - msg = await queue.get() - // message should reset in three seconds - t.ok(msg.id, 'Got a msg.id (sanity check)') - await timeout(2_000) - id = await queue.ping(msg.ack, {resetTries: true}) - t.ok(id, 'Received an id when acking this message') - // wait until the msg has returned to the queue - await timeout(6_000) - msg = await queue.get() - t.equal(msg.tries, 1, 'Tries were reset') - await queue.ack(msg.ack) - msg = await queue.get() - // no more messages - t.ok(!msg, 'No msg received') - - t.pass('Finished test ok') - t.end() - }) - - test('client.close()', function (t) { - t.pass('client.close()') - client.close() - t.end() - }) - -}) + test('ping: check a retrieved message with a ping can still be acked', async function(t) { + const queue = new MongoDbQueue(db, 'ping', {visibility: 5}); + let msg; + let id; + + id = await queue.add('Hello, World!'); + t.ok(id, 'There is an id returned when adding a message.'); + // get something now and it shouldn't be there + msg = await queue.get(); + t.ok(msg.id, 'Got this message id'); + await timeout(4000); + // ping this message so it will be kept alive longer, another 5s + id = await queue.ping(msg.ack); + t.ok(id, 'Received an id when acking this message'); + await timeout(4000); + id = await queue.ack(msg.ack); + t.ok(id, 'Received an id when acking this message'); + msg = await queue.get(); + t.ok(!msg, 'No message when getting from an empty queue'); + + t.pass('Finished test ok'); + t.end(); + }); + + test("ping: check that an acked message can't be pinged", async function(t) { + const queue = new MongoDbQueue(db, 'ping', {visibility: 5}); + let id; + + id = await queue.add('Hello, World!'); + t.ok(id, 'There is an id returned when adding a message.'); + // get something now and it shouldn't be there + const msg = await queue.get(); + t.ok(msg.id, 'Got this message id'); + // ack the message + id = await queue.ack(msg.ack); + t.ok(id, 'Received an id when acking this message'); + // ping this message, even though it has been acked + id = await queue.ping(msg.ack) + .catch((err) => t.ok(err, 'Error when pinging an acked message')); + t.ok(!id, 'Received no id when pinging an acked message'); + + t.pass('Finished test ok'); + t.end(); + }); + + test('ping: check visibility option overrides the queue visibility', async function(t) { + const queue = new MongoDbQueue(db, 'ping', {visibility: 3}); + let msg; + let id; + + id = await queue.add('Hello, World!'); + t.ok(id, 'There is an id returned when adding a message.'); + msg = await queue.get(); + // message should reset in three seconds + t.ok(msg.id, 'Got a msg.id (sanity check)'); + await timeout(2000); + // ping this message so it will be kept alive longer, another 5s instead of 3s + id = await queue.ping(msg.ack, {visibility: 5}); + t.ok(id, 'Received an id when acking this message'); + // wait 4s so the msg would normally have returns to the queue + await timeout(4000); + msg = await queue.get(); + // messages should not be back yet + t.ok(!msg, 'No msg received'); + // wait 2s so the msg should have returns to the queue + await timeout(2000); + msg = await queue.get(); + // yes, there should be a message on the queue again + t.ok(msg.id, 'Got a msg.id (sanity check)'); + await queue.ack(msg.ack); + msg = await queue.get(); + // no more messages + t.ok(!msg, 'No msg received'); + + t.pass('Finished test ok'); + t.end(); + }); + + test('ping: reset tries', async function(t) { + const queue = new MongoDbQueue(db, 'ping', {visibility: 3}); + let msg; + let id; + + id = await queue.add('Hello, World!'); + t.ok(id, 'There is an id returned when adding a message.'); + msg = await queue.get(); + // message should reset in three seconds + t.ok(msg.id, 'Got a msg.id (sanity check)'); + await timeout(2000); + id = await queue.ping(msg.ack, {resetTries: true}); + t.ok(id, 'Received an id when acking this message'); + // wait until the msg has returned to the queue + await timeout(6000); + msg = await queue.get(); + t.equal(msg.tries, 1, 'Tries were reset'); + await queue.ack(msg.ack); + msg = await queue.get(); + // no more messages + t.ok(!msg, 'No msg received'); + + t.pass('Finished test ok'); + t.end(); + }); + + test('client.close()', function(t) { + t.pass('client.close()'); + client.close(); + t.end(); + }); +}); diff --git a/test/setup.js b/test/setup.js index 90cabe1..8b97b1f 100644 --- a/test/setup.js +++ b/test/setup.js @@ -1,7 +1,7 @@ -const mongodb = require('mongodb') +const mongodb = require('mongodb'); -const url = 'mongodb://localhost:27017/' -const dbName = 'mongodb-queue' +const url = 'mongodb://localhost:27017/'; +const dbName = 'mongodb-queue'; const collections = [ 'default', @@ -16,16 +16,16 @@ const collections = [ 'dead-queue', 'queue-2', 'dead-queue-2', -] +]; module.exports = async function() { - const client = new mongodb.MongoClient(url, { useNewUrlParser: true }) + const client = new mongodb.MongoClient(url, {useNewUrlParser: true}); - await client.connect() - const db = client.db(dbName) + await client.connect(); + const db = client.db(dbName); await Promise.all( - collections.map((col) => db.collection(col).deleteMany()) - ) - return {client, db} -} + collections.map((col) => db.collection(col).deleteMany()), + ); + return {client, db}; +}; diff --git a/test/stats.js b/test/stats.js index 424984f..ef71f05 100644 --- a/test/stats.js +++ b/test/stats.js @@ -1,75 +1,67 @@ -const test = require('tape') -const {timeout} = require('./_timeout') +const test = require('tape'); +const {timeout} = require('./_timeout'); -const setup = require('./setup.js') -const MongoDbQueue = require('../').default +const setup = require('./setup.js'); +const MongoDbQueue = require('../').default; setup().then(({client, db}) => { + test('first test', function(t) { + const queue = new MongoDbQueue(db, 'stats'); + t.ok(queue, 'Queue created ok'); + t.end(); + }); - test('first test', function (t) { - const queue = new MongoDbQueue(db, 'stats') - t.ok(queue, 'Queue created ok') - t.end() - }); + test('stats for a single message added, received and acked', async function(t) { + const q = new MongoDbQueue(db, 'stats1'); - test('stats for a single message added, received and acked', async function (t) { - const q = new MongoDbQueue(db, 'stats1') - let msg - let id + const id = await q.add('Hello, World!'); + t.ok(id, 'Received an id for this message'); + t.equal(await q.total(), 1, 'Total number of messages is one'); + t.equal(await q.size(), 1, 'Size of queue is one'); + t.equal(await q.inFlight(), 0, 'There are no inFlight messages'); + t.equal(await q.done(), 0, 'There are no done messages'); + const msg = await q.get(); + t.equal(await q.total(), 1, 'Total number of messages is still one'); + t.equal(await q.size(), 0, 'Size of queue is now zero (ie. none to come)'); + t.equal(await q.inFlight(), 1, 'There is one inflight message'); + t.equal(await q.done(), 0, 'There are still no done messages'); + // now ack that message + await q.ack(msg.ack); + t.equal(await q.total(), 1, 'Total number of messages is again one'); + t.equal(await q.size(), 0, 'Size of queue is still zero (ie. none to come)'); + t.equal(await q.inFlight(), 0, 'There are no inflight messages anymore'); + t.equal(await q.done(), 1, 'There is now one processed message'); - id = await q.add('Hello, World!') - t.ok(id, 'Received an id for this message') - t.equal(await q.total(), 1, 'Total number of messages is one') - t.equal(await q.size(), 1, 'Size of queue is one') - t.equal(await q.inFlight(), 0, 'There are no inFlight messages') - t.equal(await q.done(), 0, 'There are no done messages') - msg = await q.get() - t.equal(await q.total(), 1, 'Total number of messages is still one') - t.equal(await q.size(), 0, 'Size of queue is now zero (ie. none to come)') - t.equal(await q.inFlight(), 1, 'There is one inflight message') - t.equal(await q.done(), 0, 'There are still no done messages') - // now ack that message - msg = await q.ack(msg.ack) - t.equal(await q.total(), 1, 'Total number of messages is again one') - t.equal(await q.size(), 0, 'Size of queue is still zero (ie. none to come)') - t.equal(await q.inFlight(), 0, 'There are no inflight messages anymore') - t.equal(await q.done(), 1, 'There is now one processed message') + t.end(); + }); - t.end() - }) + // ToDo: add more tests for adding a message, getting it and letting it lapse + // then re-checking all stats. + test('stats for a single message added, received, timed-out and back on queue', async function(t) { + const q = new MongoDbQueue(db, 'stats2', {visibility: 3}); - // ToDo: add more tests for adding a message, getting it and letting it lapse - // then re-checking all stats. + const id = await q.add('Hello, World!'); + t.ok(id, 'Received an id for this message'); + t.equal(await q.total(), 1, 'Total number of messages is one'); + t.equal(await q.size(), 1, 'Size of queue is one'); + t.equal(await q.inFlight(), 0, 'There are no inFlight messages'); + t.equal(await q.done(), 0, 'There are no done messages'); + // let's set one to be inFlight + await q.get(); + // msg is ignored, we don't care about the message here + await timeout(4000); + t.equal(await q.total(), 1, 'Total number of messages is still one'); + t.equal(await q.size(), 1, 'Size of queue is still at one'); + t.equal(await q.inFlight(), 0, 'There are no inflight messages again'); + t.equal(await q.done(), 0, 'There are still no done messages'); - test('stats for a single message added, received, timed-out and back on queue', async function (t) { - const q = new MongoDbQueue(db, 'stats2', {visibility: 3}) - let id - let msg - - id = await q.add('Hello, World!') - t.ok(id, 'Received an id for this message') - t.equal(await q.total(), 1, 'Total number of messages is one') - t.equal(await q.size(), 1, 'Size of queue is one') - t.equal(await q.inFlight(), 0, 'There are no inFlight messages') - t.equal(await q.done(), 0, 'There are no done messages') - // let's set one to be inFlight - msg = await q.get() - // msg is ignored, we don't care about the message here - await timeout(4_000) - t.equal(await q.total(), 1, 'Total number of messages is still one') - t.equal(await q.size(), 1, 'Size of queue is still at one') - t.equal(await q.inFlight(), 0, 'There are no inflight messages again') - t.equal(await q.done(), 0, 'There are still no done messages') - - t.end() - }) - - test('client.close()', function (t) { - t.pass('client.close()') - client.close() - t.end() - }) - -}) + t.end(); + }); + test('client.close()', function(t) { + t.pass('client.close()'); + client.close(); + t.end(); + }); +}); diff --git a/test/visibility.js b/test/visibility.js index 2596dac..778b011 100644 --- a/test/visibility.js +++ b/test/visibility.js @@ -1,93 +1,90 @@ -const test = require('tape') -const {timeout} = require('./_timeout') +const test = require('tape'); +const {timeout} = require('./_timeout'); -const setup = require('./setup.js') -const MongoDbQueue = require('../').default +const setup = require('./setup.js'); +const MongoDbQueue = require('../').default; setup().then(({client, db}) => { - - test('visibility: check message is back in queue after 3s', async function (t) { - const queue = new MongoDbQueue(db, 'visibility', {visibility: 3}) - let msg - - await queue.add('Hello, World!') - msg = await queue.get() - t.ok(msg.id, 'Got a msg.id (sanity check)') - await timeout(4_000) - msg = await queue.get() - // yes, there should be a message on the queue again - t.ok(msg.id, 'Got a msg.id (sanity check)') - await queue.ack(msg.ack) - msg = await queue.get() - t.ok(!msg, 'No msg received') - - t.pass('Finished test ok') - t.end() - }) - - test("visibility: check that a late ack doesn't remove the msg", async function (t) { - const queue = new MongoDbQueue(db, 'visibility', {visibility: 3}) - let originalAck - let msg - - await queue.add('Hello, World!') - msg = await queue.get() - t.ok(msg.id, 'Got a msg.id (sanity check)') - // remember this original ack - originalAck = msg.ack - // wait over 3s so the msg returns to the queue - await timeout(4_000) - - t.pass('Back from timeout, now acking the message') - - // now ack the message but too late - it shouldn't be deleted - msg = await queue.ack(msg.ack) - .catch((err) => t.ok(err, 'Got an error when acking the message late')) - t.ok(!msg, 'No message was updated') - msg = await queue.get() - // the message should now be able to be retrieved, with a new 'ack' id - t.ok(msg.id, 'Got a msg.id (sanity check)') - t.notEqual(msg.ack, originalAck, 'Original ack and new ack are different') - - // now ack this new retrieval - await queue.ack(msg.ack) - msg = await queue.get() - - // no more messages - t.ok(!msg, 'No msg received') - - t.pass('Finished test ok') - t.end() - }) - - test("visibility: check visibility option overrides the queue visibility", async function (t) { - const queue = new MongoDbQueue(db, 'visibility', {visibility: 2}) - let msg - - await queue.add('Hello, World!') - msg = await queue.get({visibility: 4}) - t.ok(msg.id, 'Got a msg.id (sanity check)') - // wait over 2s so the msg would normally have returns to the queue - await timeout(3_000) - msg = await queue.get() - t.ok(!msg, 'No msg received') - // wait 2s so the msg should have returns to the queue - await timeout(2_000) - msg = await queue.get() - t.ok(msg.id, 'Got a msg.id (sanity check)') - await queue.ack(msg.ack) - msg = await queue.get() - // no more messages - t.ok(!msg, 'No msg received') - - t.pass('Finished test ok') - t.end() - }) - - test('client.close()', function (t) { - t.pass('client.close()') - client.close() - t.end() - }) - -}) + test('visibility: check message is back in queue after 3s', async function(t) { + const queue = new MongoDbQueue(db, 'visibility', {visibility: 3}); + let msg; + + await queue.add('Hello, World!'); + msg = await queue.get(); + t.ok(msg.id, 'Got a msg.id (sanity check)'); + await timeout(4000); + msg = await queue.get(); + // yes, there should be a message on the queue again + t.ok(msg.id, 'Got a msg.id (sanity check)'); + await queue.ack(msg.ack); + msg = await queue.get(); + t.ok(!msg, 'No msg received'); + + t.pass('Finished test ok'); + t.end(); + }); + + test("visibility: check that a late ack doesn't remove the msg", async function(t) { + const queue = new MongoDbQueue(db, 'visibility', {visibility: 3}); + let msg; + + await queue.add('Hello, World!'); + msg = await queue.get(); + t.ok(msg.id, 'Got a msg.id (sanity check)'); + // remember this original ack + const originalAck = msg.ack; + // wait over 3s so the msg returns to the queue + await timeout(4000); + + t.pass('Back from timeout, now acking the message'); + + // now ack the message but too late - it shouldn't be deleted + msg = await queue.ack(msg.ack) + .catch((err) => t.ok(err, 'Got an error when acking the message late')); + t.ok(!msg, 'No message was updated'); + msg = await queue.get(); + // the message should now be able to be retrieved, with a new 'ack' id + t.ok(msg.id, 'Got a msg.id (sanity check)'); + t.notEqual(msg.ack, originalAck, 'Original ack and new ack are different'); + + // now ack this new retrieval + await queue.ack(msg.ack); + msg = await queue.get(); + + // no more messages + t.ok(!msg, 'No msg received'); + + t.pass('Finished test ok'); + t.end(); + }); + + test('visibility: check visibility option overrides the queue visibility', async function(t) { + const queue = new MongoDbQueue(db, 'visibility', {visibility: 2}); + let msg; + + await queue.add('Hello, World!'); + msg = await queue.get({visibility: 4}); + t.ok(msg.id, 'Got a msg.id (sanity check)'); + // wait over 2s so the msg would normally have returns to the queue + await timeout(3000); + msg = await queue.get(); + t.ok(!msg, 'No msg received'); + // wait 2s so the msg should have returns to the queue + await timeout(2000); + msg = await queue.get(); + t.ok(msg.id, 'Got a msg.id (sanity check)'); + await queue.ack(msg.ack); + msg = await queue.get(); + // no more messages + t.ok(!msg, 'No msg received'); + + t.pass('Finished test ok'); + t.end(); + }); + + test('client.close()', function(t) { + t.pass('client.close()'); + client.close(); + t.end(); + }); +}); From 1760551f1a3f8bc479cf9730f284ec0a139be1ad Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 29 Aug 2023 16:38:29 +0100 Subject: [PATCH 16/31] =?UTF-8?q?=F0=9F=91=B7=E2=80=8D=E2=99=80=EF=B8=8F?= =?UTF-8?q?=20Add=20build=20matrix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run the build against different versions of the `mongodb` driver --- .github/workflows/test.yml | 12 +++++++++++- package.json | 3 ++- test/setup.js | 2 +- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6e1cab4..6d82d27 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,9 +11,17 @@ on: jobs: build: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + mongodb: + - 4.4 + mongo_driver: + - mongodb4 + - mongodb5 services: mongodb: - image: mongo:4.4 + image: mongo:${{ matrix.mongodb }} ports: - 27017:27017 timeout-minutes: 10 @@ -40,6 +48,8 @@ jobs: run: npm run lint - name: Test run: npm test + env: + MONGO_DRIVER: ${{ matrix.mongo_driver }} - name: Tag if: ${{ github.ref == 'refs/heads/main' }} run: ./tag.sh diff --git a/package.json b/package.json index da6ab37..262cd14 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,8 @@ "devDependencies": { "@reedsy/eslint-plugin": "^0.14.2", "eslint": "^8.35.0", - "mongodb": "^5.0.0", + "mongodb4": "npm:mongodb@^4.0.0", + "mongodb5": "npm:mongodb@^5.0.0", "tape": "^4.10.1", "typescript": "^4.9.5" }, diff --git a/test/setup.js b/test/setup.js index 8b97b1f..6efb66a 100644 --- a/test/setup.js +++ b/test/setup.js @@ -1,4 +1,4 @@ -const mongodb = require('mongodb'); +const mongodb = require(process.env.MONGO_DRIVER || 'mongodb'); const url = 'mongodb://localhost:27017/'; const dbName = 'mongodb-queue'; From b69ffeee261fd95e153f5c8f7e2c5ec9c67cfa6e Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 29 Aug 2023 16:54:05 +0100 Subject: [PATCH 17/31] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Add=20support=20for?= =?UTF-8?q?=20`mongodb@6`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change adds support for [`mongodb@6`][1]. The main change that affects this library is that `findOneAndUpdate()` now returns the document itself by default, and metadata must be explicitly requested with `includeResultMetadata: true`. [1]: https://github.com/mongodb/node-mongodb-native/releases/tag/v6.0.0 --- .github/workflows/test.yml | 1 + mongodb-queue.ts | 15 +++++++++------ package.json | 5 +++-- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6d82d27..c8d168d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,6 +19,7 @@ jobs: mongo_driver: - mongodb4 - mongodb5 + - mongodb6 services: mongodb: image: mongo:${{ matrix.mongodb }} diff --git a/mongodb-queue.ts b/mongodb-queue.ts index 82db763..d61e8dd 100644 --- a/mongodb-queue.ts +++ b/mongodb-queue.ts @@ -139,10 +139,11 @@ export default class Queue { visible: nowPlusSecs(visibility), }, }; - const options: FindOneAndUpdateOptions = { + const options = { sort: sort, returnDocument: 'after', - }; + includeResultMetadata: true, + } satisfies FindOneAndUpdateOptions; const result = await this.col.findOneAndUpdate(query, update, options); const msg = result.value as WithId>; @@ -183,9 +184,10 @@ export default class Queue { visible: nowPlusSecs(visibility), }, }; - const options: FindOneAndUpdateOptions = { + const options = { returnDocument: 'after', - }; + includeResultMetadata: true, + } satisfies FindOneAndUpdateOptions; if (opts.resetTries) { update.$set = { @@ -212,9 +214,10 @@ export default class Queue { deleted: now(), }, }; - const options: FindOneAndUpdateOptions = { + const options = { returnDocument: 'after', - }; + includeResultMetadata: true, + } satisfies FindOneAndUpdateOptions; const msg = await this.col.findOneAndUpdate(query, update, options); if (!msg.value) { throw new Error('Queue.ack(): Unidentified ack : ' + ack); diff --git a/package.json b/package.json index 262cd14..15a8ba3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@reedsy/mongodb-queue", - "version": "5.0.1", + "version": "5.1.0", "description": "Message queues which uses MongoDB.", "main": "mongodb-queue.js", "scripts": { @@ -15,11 +15,12 @@ "eslint": "^8.35.0", "mongodb4": "npm:mongodb@^4.0.0", "mongodb5": "npm:mongodb@^5.0.0", + "mongodb6": "npm:mongodb@^6.0.0", "tape": "^4.10.1", "typescript": "^4.9.5" }, "peerDependencies": { - "mongodb": "^4.0.0 || ^5.0.0" + "mongodb": "^4.0.0 || ^5.0.0 || ^6.0.0" }, "homepage": "https://github.com/chilts/mongodb-queue", "repository": { From 9a5c893928c1f106da19e604119f7578c0d373fc Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Mon, 6 Nov 2023 16:25:40 +0000 Subject: [PATCH 18/31] =?UTF-8?q?=F0=9F=91=B7=E2=80=8D=E2=99=80=EF=B8=8F?= =?UTF-8?q?=20Build=20against=20MongoDB=20v5.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MongoDB 4.4 will be [end-of-lifed in February][1]. This change runs the build against MongoDB 5.0 in readiness for upgrading our Staging and Production replica sets. [1]: https://www.mongodb.com/support-policy/lifecycles --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c8d168d..a596179 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,6 +16,7 @@ jobs: matrix: mongodb: - 4.4 + - 5.0 mongo_driver: - mongodb4 - mongodb5 From 772580d96e1ffdc23e1cac98f58fb0a5c7967a45 Mon Sep 17 00:00:00 2001 From: Dawid Kisielewski Date: Tue, 2 Jan 2024 11:23:28 +0100 Subject: [PATCH 19/31] =?UTF-8?q?=F0=9F=90=9B=20Remove=20undefined=20props?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At the moment when we pass payload like this: ```ts {someProp: undefined} ``` It stores it mongoDb Db, but it auto convert it to null value, however it might be error prone as we may enqueue job that has `callbackUrl: undefined`, then the mongo queue saves in db, which converts it to `callbackUrl: null`, then when getting the message from queue worker validate the job payload and has type `t.partial({callbackUrl: t.string}})`. The validation will fail. --- mongodb-queue.ts | 2 +- package.json | 2 +- test/default.js | 11 +++++++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/mongodb-queue.ts b/mongodb-queue.ts index d61e8dd..82a1e44 100644 --- a/mongodb-queue.ts +++ b/mongodb-queue.ts @@ -118,7 +118,7 @@ export default class Queue { }); } - const results = await this.col.insertMany(msgs); + const results = await this.col.insertMany(msgs, {ignoreUndefined: true}); if (payload instanceof Array) return '' + results.insertedIds; return '' + results.insertedIds[0]; } diff --git a/package.json b/package.json index 15a8ba3..5d1549d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@reedsy/mongodb-queue", - "version": "5.1.0", + "version": "6.0.0", "description": "Message queues which uses MongoDB.", "main": "mongodb-queue.js", "scripts": { diff --git a/test/default.js b/test/default.js index 9af5281..03e0370 100644 --- a/test/default.js +++ b/test/default.js @@ -56,6 +56,17 @@ setup().then(({client, db}) => { t.end(); }); + test('remove undefined properties', async function(t) { + const queue = new MongoDbQueue(db, 'default'); + const id = await queue.add({text: 'Hello, World!', undefinedProp: undefined}); + t.ok(id, 'Received an id for this message'); + + const msg = await queue.get(); + t.ok(msg.id, 'Got a msg.id'); + t.equal('undefinedProp' in msg.payload, false, 'Payload has undefinedProp and it should be removed'); + t.end(); + }); + test('client.close()', function(t) { t.pass('client.close()'); client.close(); From b0df39ea22058e8e0e4decc02e86d8c734492eb3 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Thu, 18 Jan 2024 11:46:58 +0000 Subject: [PATCH 20/31] =?UTF-8?q?=F0=9F=91=B7=E2=80=8D=E2=99=80=EF=B8=8F?= =?UTF-8?q?=20Move=20publishing=20inside=20single=20workflow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At the moment, we have two Github Action workflows: - `test.yml`: runs build and test, then tags when bumping the version in `main` - `publish.yml`: releases the package when a new tag is published The issue with this setup is that the built-in `GITHUB_TOKEN` [will not trigger another workflow][1], so we had to add a separate PAT with write permissions to our repos, which was a bit of a security concern. In order to avoid the need for this extra token, with its associated risks and administrative overheads (like rotating), this change combines our workflows into a single workflow. We tweak the `tag.sh` to `release.sh`, and it's now also in charge of publishing (since it knows when we've pushed a new tag). [1]: https://docs.github.com/en/actions/using-workflows/triggering-a-workflow#triggering-a-workflow-from-a-workflow --- .github/workflows/ci.yml | 48 +++++++++++++++++++++++++++++ .github/workflows/publish.yml | 30 ------------------ .github/workflows/test.yml | 57 ----------------------------------- tag.sh => release.sh | 2 ++ 4 files changed, 50 insertions(+), 87 deletions(-) create mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/publish.yml delete mode 100644 .github/workflows/test.yml rename tag.sh => release.sh (96%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..ef6319d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,48 @@ +name: CI + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + mongodb: + - 4.4 + - 5.0 + mongo_driver: + - mongodb4 + - mongodb5 + - mongodb6 + services: + mongodb: + image: mongo:${{ matrix.mongodb }} + ports: + - 27017:27017 + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '20.x' + registry-url: 'https://npm.pkg.github.com' + - name: Install + run: npm install + - name: Lint + run: npm run lint + - name: Test + run: npm test + env: + MONGO_DRIVER: ${{ matrix.mongo_driver }} + - name: Release + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + run: ./release.sh + env: + NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml deleted file mode 100644 index 8775feb..0000000 --- a/.github/workflows/publish.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Publish - -on: - push: - tags: - - '*' - -jobs: - build: - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: '18.x' - registry-url: 'https://npm.pkg.github.com' - - name: Install - # Skip post-install to avoid malicious scripts stealing PAT - run: npm install --ignore-script - env: - # GITHUB_TOKEN can't access packages hosted in private repos, - # even within the same organisation - NODE_AUTH_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - - name: Post-install - run: npm rebuild && npm run prepare --if-present - - name: Publish - run: npm publish - env: - NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index a596179..0000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: Test - -on: - push: - branches: - - main - pull_request: - branches: - - main - -jobs: - build: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - mongodb: - - 4.4 - - 5.0 - mongo_driver: - - mongodb4 - - mongodb5 - - mongodb6 - services: - mongodb: - image: mongo:${{ matrix.mongodb }} - ports: - - 27017:27017 - timeout-minutes: 10 - steps: - - uses: actions/checkout@v3 - with: - # Use PAT instead of default Github token, because the default - # token deliberately will not trigger another workflow run - token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - - uses: actions/setup-node@v3 - with: - node-version: '18.x' - registry-url: 'https://npm.pkg.github.com' - - name: Install - # Skip post-install to avoid malicious scripts stealing PAT - run: npm install --ignore-script - env: - # GITHUB_TOKEN can't access packages hosted in private repos, - # even within the same organisation - NODE_AUTH_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - - name: Post-install - run: npm rebuild && npm run prepare --if-present - - name: Lint - run: npm run lint - - name: Test - run: npm test - env: - MONGO_DRIVER: ${{ matrix.mongo_driver }} - - name: Tag - if: ${{ github.ref == 'refs/heads/main' }} - run: ./tag.sh diff --git a/tag.sh b/release.sh similarity index 96% rename from tag.sh rename to release.sh index 9f544d5..6166f39 100755 --- a/tag.sh +++ b/release.sh @@ -18,3 +18,5 @@ fi git tag $VERSION git push origin refs/tags/$VERSION + +npm publish From 4e7cf41fa94ef8a0bf333c2c9d742abb3442e624 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Thu, 18 Jan 2024 11:57:13 +0000 Subject: [PATCH 21/31] =?UTF-8?q?=F0=9F=91=B7=E2=80=8D=E2=99=80=EF=B8=8F?= =?UTF-8?q?=20Use=20`GITHUB=5FTOKEN`=20for=20install?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ef6319d..31e95c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,12 @@ jobs: node-version: '20.x' registry-url: 'https://npm.pkg.github.com' - name: Install - run: npm install + # Skip post-install to avoid malicious scripts stealing PAT + run: npm install --ignore-script + env: + NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Post-install + run: npm rebuild && npm run prepare --if-present - name: Lint run: npm run lint - name: Test From 860daf817b98d9c45b12aa03e29c6e56d0005dc0 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 7 May 2024 16:37:07 +0100 Subject: [PATCH 22/31] =?UTF-8?q?=F0=9F=91=B7=E2=80=8D=E2=99=80=EF=B8=8F?= =?UTF-8?q?=20Run=20against=20MongoDB=20v6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 31e95c8..54c8113 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,11 +15,8 @@ jobs: fail-fast: false matrix: mongodb: - - 4.4 - - 5.0 + - '6.0' mongo_driver: - - mongodb4 - - mongodb5 - mongodb6 services: mongodb: From f58f61f70f19aaf560b4ce5bf8a51cbf98b2639c Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 7 May 2024 16:38:18 +0100 Subject: [PATCH 23/31] =?UTF-8?q?=F0=9F=91=B7=E2=80=8D=E2=99=80=EF=B8=8F?= =?UTF-8?q?=20Add=20`success`=20job?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 54c8113..11fe184 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,3 +48,14 @@ jobs: run: ./release.sh env: NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # A silly job just so we can mark this as the required check for PRs + # instead of maintaining the full list of matrix jobs + success: + needs: + - build + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Success + run: echo "Success" From 24d10d5183977d099f2622be9aaad9295ca09d74 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 7 May 2024 16:35:37 +0100 Subject: [PATCH 24/31] =?UTF-8?q?=F0=9F=92=A5=20Remove=20default=20exports?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Default exports have [bad interoperability][1] between Common JS and ES Module imports, so the safest thing to do is just remove them. This is a **BREAKING** change which removes our export default statements, and adds a linter rule to prevent us from adding them. [1]: https://github.com/evanw/esbuild/issues/1719#issuecomment-953470495 --- .eslintrc.yml | 10 ++++++++++ mongodb-queue.ts | 6 +++--- package.json | 2 +- test/clean.js | 4 ++-- test/dead-queue.js | 12 ++++++------ test/default.js | 10 +++++----- test/delay.js | 6 +++--- test/indexes.js | 4 ++-- test/many.js | 6 +++--- test/multi.js | 4 ++-- test/ping.js | 10 +++++----- test/stats.js | 8 ++++---- test/visibility.js | 8 ++++---- 13 files changed, 50 insertions(+), 40 deletions(-) diff --git a/.eslintrc.yml b/.eslintrc.yml index 08f1c63..f1108bb 100644 --- a/.eslintrc.yml +++ b/.eslintrc.yml @@ -4,3 +4,13 @@ parserOptions: project: './tsconfig.json' env: node: true +rules: + # default exports have bad CJS/ESM interoperability + no-restricted-exports: + - error + - restrictDefaultExports: + direct: true + named: true + defaultFrom: true + namedFrom: true + namespaceFrom: true diff --git a/mongodb-queue.ts b/mongodb-queue.ts index 82a1e44..baa2dee 100644 --- a/mongodb-queue.ts +++ b/mongodb-queue.ts @@ -28,7 +28,7 @@ function nowPlusSecs(secs: number): string { export type QueueOptions = { visibility?: number; delay?: number; - deadQueue?: Queue; + deadQueue?: MongoDBQueue; maxRetries?: number; }; @@ -63,12 +63,12 @@ export type ExternalMessage = { tries: number; }; -export default class Queue { +export class MongoDBQueue { private readonly col: Collection>>; private readonly visibility: number; private readonly delay: number; private readonly maxRetries: number; - private readonly deadQueue: Queue; + private readonly deadQueue: MongoDBQueue; public constructor(db: Db, name: string, opts: QueueOptions = {}) { if (!db) { diff --git a/package.json b/package.json index 5d1549d..890c71a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@reedsy/mongodb-queue", - "version": "6.0.0", + "version": "7.0.0", "description": "Message queues which uses MongoDB.", "main": "mongodb-queue.js", "scripts": { diff --git a/test/clean.js b/test/clean.js index 21d64a1..d034865 100644 --- a/test/clean.js +++ b/test/clean.js @@ -1,11 +1,11 @@ const test = require('tape'); const setup = require('./setup.js'); -const MongoDbQueue = require('../').default; +const {MongoDBQueue} = require('../'); setup().then(({client, db}) => { test('clean: check deleted messages are deleted', async function(t) { - const q = new MongoDbQueue(db, 'clean', {visibility: 3}); + const q = new MongoDBQueue(db, 'clean', {visibility: 3}); t.equal(await q.size(), 0, 'There is currently nothing on the queue'); t.equal(await q.total(), 0, 'There is currently nothing in the queue at all'); diff --git a/test/dead-queue.js b/test/dead-queue.js index d32a1eb..ca4c720 100644 --- a/test/dead-queue.js +++ b/test/dead-queue.js @@ -1,18 +1,18 @@ const test = require('tape'); const setup = require('./setup.js'); -const MongoDbQueue = require('../').default; +const {MongoDBQueue} = require('../'); setup().then(({client, db}) => { test('first test', function(t) { - const queue = new MongoDbQueue(db, 'queue', {visibility: 3, deadQueue: 'dead-queue'}); + const queue = new MongoDBQueue(db, 'queue', {visibility: 3, deadQueue: 'dead-queue'}); t.ok(queue, 'Queue created ok'); t.end(); }); test('single message going over 5 tries, should appear on dead-queue', async function(t) { - const deadQueue = new MongoDbQueue(db, 'dead-queue'); - const queue = new MongoDbQueue(db, 'queue', {visibility: 1, deadQueue: deadQueue}); + const deadQueue = new MongoDBQueue(db, 'dead-queue'); + const queue = new MongoDBQueue(db, 'queue', {visibility: 1, deadQueue: deadQueue}); let msg; const origId = await queue.add('Hello, World!'); @@ -41,8 +41,8 @@ setup().then(({client, db}) => { }); test('two messages, with first going over 3 tries', async function(t) { - const deadQueue = new MongoDbQueue(db, 'dead-queue-2'); - const queue = new MongoDbQueue(db, 'queue-2', {visibility: 1, deadQueue: deadQueue, maxRetries: 3}); + const deadQueue = new MongoDBQueue(db, 'dead-queue-2'); + const queue = new MongoDBQueue(db, 'queue-2', {visibility: 1, deadQueue: deadQueue, maxRetries: 3}); let msg; const origId = await queue.add('Hello, World!'); diff --git a/test/default.js b/test/default.js index 03e0370..75d1603 100644 --- a/test/default.js +++ b/test/default.js @@ -1,17 +1,17 @@ const test = require('tape'); const setup = require('./setup.js'); -const MongoDbQueue = require('../').default; +const {MongoDBQueue} = require('../'); setup().then(({client, db}) => { test('first test', function(t) { - const queue = new MongoDbQueue(db, 'default'); + const queue = new MongoDBQueue(db, 'default'); t.ok(queue, 'Queue created ok'); t.end(); }); test('single round trip', async function(t) { - const queue = new MongoDbQueue(db, 'default'); + const queue = new MongoDBQueue(db, 'default'); let id; id = await queue.add('Hello, World!'); @@ -33,7 +33,7 @@ setup().then(({client, db}) => { }); test("single round trip, can't be acked again", async function(t) { - const queue = new MongoDbQueue(db, 'default'); + const queue = new MongoDBQueue(db, 'default'); let id; id = await queue.add('Hello, World!'); @@ -57,7 +57,7 @@ setup().then(({client, db}) => { }); test('remove undefined properties', async function(t) { - const queue = new MongoDbQueue(db, 'default'); + const queue = new MongoDBQueue(db, 'default'); const id = await queue.add({text: 'Hello, World!', undefinedProp: undefined}); t.ok(id, 'Received an id for this message'); diff --git a/test/delay.js b/test/delay.js index 0dc7af6..bafe991 100644 --- a/test/delay.js +++ b/test/delay.js @@ -1,12 +1,12 @@ const test = require('tape'); const setup = require('./setup.js'); -const MongoDbQueue = require('../').default; +const {MongoDBQueue} = require('../'); const {timeout} = require('./_timeout.js'); setup().then(({client, db}) => { test('delay: check messages on this queue are returned after the delay', async function(t) { - const queue = new MongoDbQueue(db, 'delay', {delay: 3}); + const queue = new MongoDBQueue(db, 'delay', {delay: 3}); let msg; const id = await queue.add('Hello, World!'); @@ -27,7 +27,7 @@ setup().then(({client, db}) => { }); test('delay: check an individual message delay overrides the queue delay', async function(t) { - const queue = new MongoDbQueue(db, 'delay'); + const queue = new MongoDBQueue(db, 'delay'); let msg; const id = await queue.add('I am delayed by 3 seconds', {delay: 3}); diff --git a/test/indexes.js b/test/indexes.js index 791932f..1fa50e6 100644 --- a/test/indexes.js +++ b/test/indexes.js @@ -1,11 +1,11 @@ const test = require('tape'); const setup = require('./setup.js'); -const MongoDbQueue = require('../').default; +const {MongoDBQueue} = require('../'); setup().then(({client, db}) => { test('visibility: check message is back in queue after 3s', async function(t) { - const queue = new MongoDbQueue(db, 'visibility', {visibility: 3}); + const queue = new MongoDBQueue(db, 'visibility', {visibility: 3}); await queue.createIndexes(); t.pass('Indexes created'); diff --git a/test/many.js b/test/many.js index b52fa2f..143e32d 100644 --- a/test/many.js +++ b/test/many.js @@ -1,13 +1,13 @@ const test = require('tape'); const setup = require('./setup.js'); -const MongoDbQueue = require('../').default; +const {MongoDBQueue} = require('../'); const total = 250; setup().then(({client, db}) => { test('many: add ' + total + ' messages, get ' + total + ' back', async function(t) { - const queue = new MongoDbQueue(db, 'many'); + const queue = new MongoDBQueue(db, 'many'); const msgs = []; const msgsToQueue = []; @@ -36,7 +36,7 @@ setup().then(({client, db}) => { }); test('many: add no messages, receive err in callback', async function(t) { - const queue = new MongoDbQueue(db, 'many'); + const queue = new MongoDBQueue(db, 'many'); await queue.add([]) .catch(() => t.pass('got error')); t.end(); diff --git a/test/multi.js b/test/multi.js index 8508a85..d835ecf 100644 --- a/test/multi.js +++ b/test/multi.js @@ -1,13 +1,13 @@ const test = require('tape'); const setup = require('./setup.js'); -const MongoDbQueue = require('../').default; +const {MongoDBQueue} = require('../'); const total = 250; setup().then(({client, db}) => { test('multi: add ' + total + ' messages, get ' + total + ' back', async function(t) { - const queue = new MongoDbQueue(db, 'multi'); + const queue = new MongoDBQueue(db, 'multi'); const msgs = []; for (let i = 0; i < total; i++) await queue.add('no=' + i); diff --git a/test/ping.js b/test/ping.js index 5c5a5d1..54ec2f8 100644 --- a/test/ping.js +++ b/test/ping.js @@ -2,11 +2,11 @@ const test = require('tape'); const {timeout} = require('./_timeout'); const setup = require('./setup.js'); -const MongoDbQueue = require('../').default; +const {MongoDBQueue} = require('../'); setup().then(({client, db}) => { test('ping: check a retrieved message with a ping can still be acked', async function(t) { - const queue = new MongoDbQueue(db, 'ping', {visibility: 5}); + const queue = new MongoDBQueue(db, 'ping', {visibility: 5}); let msg; let id; @@ -30,7 +30,7 @@ setup().then(({client, db}) => { }); test("ping: check that an acked message can't be pinged", async function(t) { - const queue = new MongoDbQueue(db, 'ping', {visibility: 5}); + const queue = new MongoDBQueue(db, 'ping', {visibility: 5}); let id; id = await queue.add('Hello, World!'); @@ -51,7 +51,7 @@ setup().then(({client, db}) => { }); test('ping: check visibility option overrides the queue visibility', async function(t) { - const queue = new MongoDbQueue(db, 'ping', {visibility: 3}); + const queue = new MongoDBQueue(db, 'ping', {visibility: 3}); let msg; let id; @@ -84,7 +84,7 @@ setup().then(({client, db}) => { }); test('ping: reset tries', async function(t) { - const queue = new MongoDbQueue(db, 'ping', {visibility: 3}); + const queue = new MongoDBQueue(db, 'ping', {visibility: 3}); let msg; let id; diff --git a/test/stats.js b/test/stats.js index ef71f05..3ee7fb8 100644 --- a/test/stats.js +++ b/test/stats.js @@ -2,17 +2,17 @@ const test = require('tape'); const {timeout} = require('./_timeout'); const setup = require('./setup.js'); -const MongoDbQueue = require('../').default; +const {MongoDBQueue} = require('../'); setup().then(({client, db}) => { test('first test', function(t) { - const queue = new MongoDbQueue(db, 'stats'); + const queue = new MongoDBQueue(db, 'stats'); t.ok(queue, 'Queue created ok'); t.end(); }); test('stats for a single message added, received and acked', async function(t) { - const q = new MongoDbQueue(db, 'stats1'); + const q = new MongoDBQueue(db, 'stats1'); const id = await q.add('Hello, World!'); t.ok(id, 'Received an id for this message'); @@ -39,7 +39,7 @@ setup().then(({client, db}) => { // then re-checking all stats. test('stats for a single message added, received, timed-out and back on queue', async function(t) { - const q = new MongoDbQueue(db, 'stats2', {visibility: 3}); + const q = new MongoDBQueue(db, 'stats2', {visibility: 3}); const id = await q.add('Hello, World!'); t.ok(id, 'Received an id for this message'); diff --git a/test/visibility.js b/test/visibility.js index 778b011..a5cd631 100644 --- a/test/visibility.js +++ b/test/visibility.js @@ -2,11 +2,11 @@ const test = require('tape'); const {timeout} = require('./_timeout'); const setup = require('./setup.js'); -const MongoDbQueue = require('../').default; +const {MongoDBQueue} = require('../'); setup().then(({client, db}) => { test('visibility: check message is back in queue after 3s', async function(t) { - const queue = new MongoDbQueue(db, 'visibility', {visibility: 3}); + const queue = new MongoDBQueue(db, 'visibility', {visibility: 3}); let msg; await queue.add('Hello, World!'); @@ -25,7 +25,7 @@ setup().then(({client, db}) => { }); test("visibility: check that a late ack doesn't remove the msg", async function(t) { - const queue = new MongoDbQueue(db, 'visibility', {visibility: 3}); + const queue = new MongoDBQueue(db, 'visibility', {visibility: 3}); let msg; await queue.add('Hello, World!'); @@ -59,7 +59,7 @@ setup().then(({client, db}) => { }); test('visibility: check visibility option overrides the queue visibility', async function(t) { - const queue = new MongoDbQueue(db, 'visibility', {visibility: 2}); + const queue = new MongoDBQueue(db, 'visibility', {visibility: 2}); let msg; await queue.add('Hello, World!'); From c44f97482b76c9d41640918a9d501e64aa90f612 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Wed, 23 Oct 2024 14:24:26 +0100 Subject: [PATCH 25/31] =?UTF-8?q?=E2=9A=A1=EF=B8=8F=20Sort=20`get()`=20by?= =?UTF-8?q?=20`visible`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At the moment, when calling `get()`, we: - query on `deleted` and `visible` - sort on `_id` We have indexes for both of these things, but separately, which results in an inefficient query, since MongoDB will have to check both indexes and can result in a complete index scan, which isn't particularly great. We sort by `_id` presumably to get the oldest job (since `_id` is an `ObjectId` whose sort order correlates to creation time). However, we probably actually want the job that was visible first. This change updates to sort to use `visible`, which also means that the query and sort can use the same index. --- mongodb-queue.ts | 2 +- package.json | 2 +- test/dead-queue.js | 31 ------------------------------- 3 files changed, 2 insertions(+), 33 deletions(-) diff --git a/mongodb-queue.ts b/mongodb-queue.ts index baa2dee..79c0563 100644 --- a/mongodb-queue.ts +++ b/mongodb-queue.ts @@ -130,7 +130,7 @@ export class MongoDBQueue { visible: {$lte: now()}, }; const sort: Sort = { - _id: 1, + visible: 1, }; const update: UpdateFilter> = { $inc: {tries: 1}, diff --git a/package.json b/package.json index 890c71a..28c1d30 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@reedsy/mongodb-queue", - "version": "7.0.0", + "version": "7.0.1", "description": "Message queues which uses MongoDB.", "main": "mongodb-queue.js", "scripts": { diff --git a/test/dead-queue.js b/test/dead-queue.js index ca4c720..f81ffea 100644 --- a/test/dead-queue.js +++ b/test/dead-queue.js @@ -40,37 +40,6 @@ setup().then(({client, db}) => { t.end(); }); - test('two messages, with first going over 3 tries', async function(t) { - const deadQueue = new MongoDBQueue(db, 'dead-queue-2'); - const queue = new MongoDBQueue(db, 'queue-2', {visibility: 1, deadQueue: deadQueue, maxRetries: 3}); - let msg; - - const origId = await queue.add('Hello, World!'); - t.ok(origId, 'Received an id for this message'); - const origId2 = await queue.add('Part II'); - t.ok(origId2, 'Received an id for this message'); - - for (let i = 1; i <= 3; i++) { - msg = await queue.get(); - t.equal(msg.id, origId, 'We return the first message on first go'); - await new Promise((resolve) => setTimeout(function() { - t.pass(`Expiration #${i}`); - resolve(); - }, 2 * 1000)); - } - - msg = await queue.get(); - t.equal(msg.id, origId2, 'Got the ID of the 2nd message'); - t.equal(msg.payload, 'Part II', 'Got the same payload as the 2nd message'); - - msg = await deadQueue.get(); - t.ok(msg.id, 'Got a message id from the deadQueue'); - t.equal(msg.payload.id, origId, 'Got the same message id as the original message'); - t.equal(msg.payload.payload, 'Hello, World!', 'Got the same as the original message'); - t.equal(msg.payload.tries, 4, 'Got the tries as 4'); - t.end(); - }); - test('client.close()', function(t) { t.pass('client.close()'); client.close(); From a2a51bd4d56f3ee1445f7f30bf0debd3c5513168 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Wed, 4 Dec 2024 15:32:49 +0000 Subject: [PATCH 26/31] =?UTF-8?q?=F0=9F=97=83=20Change=20`ack`=20to=20(str?= =?UTF-8?q?ing)=20`ObjectId`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At the moment, the `ack` is a completely random hex string. This change updates it to use an `ObjectId`, which: - encodes information about the time the job was pulled (for the first time) - ...and can therefore be used to sort jobs in the order they were originally pulled from the queue - allows us to simplify the dependencies and code a little --- mongodb-queue.ts | 9 ++------- package.json | 2 +- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/mongodb-queue.ts b/mongodb-queue.ts index 79c0563..25ba41b 100644 --- a/mongodb-queue.ts +++ b/mongodb-queue.ts @@ -10,12 +10,7 @@ * **/ -import {randomBytes} from 'crypto'; -import {Collection, Db, Filter, FindOneAndUpdateOptions, Sort, UpdateFilter, WithId} from 'mongodb'; - -function id(): string { - return randomBytes(16).toString('hex'); -} +import {Collection, Db, Filter, FindOneAndUpdateOptions, ObjectId, Sort, UpdateFilter, WithId} from 'mongodb'; function now(): string { return (new Date()).toISOString(); @@ -135,7 +130,7 @@ export class MongoDBQueue { const update: UpdateFilter> = { $inc: {tries: 1}, $set: { - ack: id(), + ack: new ObjectId().toHexString(), visible: nowPlusSecs(visibility), }, }; diff --git a/package.json b/package.json index 28c1d30..bd254b5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@reedsy/mongodb-queue", - "version": "7.0.1", + "version": "7.1.0", "description": "Message queues which uses MongoDB.", "main": "mongodb-queue.js", "scripts": { From 2822f743fe09c0af27f04af10f250f708f9653c1 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Thu, 6 Feb 2025 14:30:02 +0000 Subject: [PATCH 27/31] =?UTF-8?q?=F0=9F=97=83=20Remove=20`visible`=20when?= =?UTF-8?q?=20acking=20jobs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a non-breaking change that forms part of the migration path to improving our performance in https://github.com/reedsy/mongodb-queue/pull/15 When acking a job, we now unset the `visible` property, which makes `visible` and `deleted` mutually exclusive properties, so that the presence of one implies the absence of the other. We can do this, because the only time we query on `visible` is when we *also* query for `deleted: {$exists: false}`. Similarly, the only times we query for `deleted: {$exists: true}` are times when we don't query `visible`. --- mongodb-queue.ts | 3 +++ package.json | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/mongodb-queue.ts b/mongodb-queue.ts index 25ba41b..ce91682 100644 --- a/mongodb-queue.ts +++ b/mongodb-queue.ts @@ -208,6 +208,9 @@ export class MongoDBQueue { $set: { deleted: now(), }, + $unset: { + visible: 1, + }, }; const options = { returnDocument: 'after', diff --git a/package.json b/package.json index bd254b5..bab7c97 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@reedsy/mongodb-queue", - "version": "7.1.0", + "version": "7.1.1", "description": "Message queues which uses MongoDB.", "main": "mongodb-queue.js", "scripts": { From a67c70a64111dcbd404c11626767d12d1639bcd5 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Mon, 10 Feb 2025 11:31:57 +0000 Subject: [PATCH 28/31] =?UTF-8?q?=F0=9F=92=A5=20Change=20`deleted`=20from?= =?UTF-8?q?=20`string`=20to=20`Date`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a **BREAKING** change that will change the `deleted` field from a `string` to a `Date`, which will let us set up a TTL index on it. Although this change is technically breaking, it should be deployable with no impact on Production, since we only ever query `deleted` using the `$exists` operator. In order to leverage the TTL index, we'll need to: 1. Deploy this change 2. Migrate all existing `deleted` fields to `Date`: ```js db.collection.update( {deleted: {$type: "string"}}, [{$set: {deleted: {$toDate: "$deleted"}}}] ) ``` 3. Change the existing `deleted` index to TTL: ```js db.runCommand({ collMod: '...', index: { name: 'deleted_1', expireAfterSeconds: 2592000, // ~ 1 month } }) ``` --- mongodb-queue.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongodb-queue.ts b/mongodb-queue.ts index ce91682..25d1995 100644 --- a/mongodb-queue.ts +++ b/mongodb-queue.ts @@ -48,7 +48,7 @@ export type BaseMessage = { export type Message = BaseMessage & { ack: string; tries: number; - deleted?: string; + deleted?: Date; }; export type ExternalMessage = { @@ -206,7 +206,7 @@ export class MongoDBQueue { }; const update: UpdateFilter> = { $set: { - deleted: now(), + deleted: new Date(), }, $unset: { visible: 1, From 795a8d986d6acd5ebc6dca4c154e1a45b2b84155 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Thu, 6 Feb 2025 14:53:18 +0000 Subject: [PATCH 29/31] =?UTF-8?q?=F0=9F=92=A5=20Improve=20database=20perfo?= =?UTF-8?q?rmance?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a **BREAKING CHANGE** that aims to improve database performance by changing the queries and indexes used to perform operations. There will be no changes visible to consumers of the JavaScript API. The break will: - require consumers to run a migration query before upgrading - add new indexes - allow dropping of an old index More details on migration are at the bottom of this commit message. Motivation ---------- This library seems to have been written with the assumption that its collections are small. However, we have hundreds of thousands of jobs in various queues on Production, which causes some slow queries because of the design choices made in the schema of this library. In particular, we aim to address two issues: 1. `{deleted: {$exists: boolean}}` calls are inefficient, but used by practically every query in this library 2. counting inflight jobs has awful performance when there are many available jobs The result is that no further filtering is needed beyond the index on any of the issued queries. `$exists` --------- MongoDB's `$exists` operator has [tricky index performance][1]. In the best cases, `$exists: true` can use the index, but only if it's sparse, and `$exists: false` can **never** just use the index: it always needs to fetch documents. In order to avoid the constant use of `$exists` in this library, we rely on a logical paradigm shift: we `$unset` `visible` when acking the job, so that `visible` and `deleted` are mutually exclusive fields. Therefore we can: - add a sparse index for both of these fields - query on the field we care about, and **know** that it implies the absence of the other field, allowing the removal of `$exists` assertions It should be noted that in local testing, I observed [strange behaviour][2] when trying to use this partial index: we have to use `ack: {$gt: ''}` instead of `ack: {$exists: true}` to get MongoDB to leverage this index for some reason. `inFlight()` ------------ The existing `inFlight()` query has particularly bad performance in cases where there are many (hundreds of thousands) of jobs available to pick up. This is because the existing query uses the `deleted_1_visible_1` index, but even after filtering by `deleted` and `visible` with the index, the database will need to fetch every single job that could be picked up, and check for `ack`, which is very slow. We improve the performance here by: - the removal of the `$exists` query (see above) - the addition of a partial index that only contains unacked jobs that have been retrieved at some point by `get()`. We can then filter these by the current time to find in-flight jobs Migration path -------------- This performance improvement is built upon a shift in the assumptions made about underlying job structure: namely, that `deleted` and `visible` are now mutually exclusive properties (which was not true before). 1. Bump patch version to [`7.1.1`][3]: this will start removing the `visible` property from acked jobs in a non-breaking way 2. Deploy the patch to Production 3. Update any existing documents to match this new schema: ```js db.collection.updateMany( {deleted: {$exists: true}}, {$unset: {visible: 1}}, ) ``` 4. Bump major version to `8.0.0` and deploy 5. Drop old index `delted_1_visible_1`, which is no longer used [1]: https://www.mongodb.com/docs/manual/reference/operator/query/exists/#use-a-sparse-index-to-improve--exists-performance [2]: https://www.mongodb.com/community/forums/t/partial-index-is-not-used-during-search/290507/2 [3]: https://github.com/reedsy/mongodb-queue/pull/16 --- mongodb-queue.ts | 20 +++++++++++++------- package.json | 2 +- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/mongodb-queue.ts b/mongodb-queue.ts index 25d1995..be44b59 100644 --- a/mongodb-queue.ts +++ b/mongodb-queue.ts @@ -85,9 +85,17 @@ export class MongoDBQueue { public async createIndexes(): Promise { await Promise.all([ - this.col.createIndex({deleted: 1, visible: 1}), + this.col.createIndex({visible: 1}, {sparse: true}), this.col.createIndex({ack: 1}, {unique: true, sparse: true}), this.col.createIndex({deleted: 1}, {sparse: true}), + + // Index for efficient counts on in-flight + this.col.createIndex({visible: 1, ack: 1}, { + partialFilterExpression: { + visible: {$exists: true}, + ack: {$exists: true}, + }, + }), ]); } @@ -121,7 +129,6 @@ export class MongoDBQueue { public async get(opts: GetOptions = {}): Promise | null> { const visibility = opts.visibility || this.visibility; const query: Filter>> = { - deleted: {$exists: false}, visible: {$lte: now()}, }; const sort: Sort = { @@ -172,7 +179,6 @@ export class MongoDBQueue { const query: Filter>> = { ack: ack, visible: {$gt: now()}, - deleted: {$exists: false}, }; const update: UpdateFilter> = { $set: { @@ -202,7 +208,6 @@ export class MongoDBQueue { const query: Filter>> = { ack: ack, visible: {$gt: now()}, - deleted: {$exists: false}, }; const update: UpdateFilter> = { $set: { @@ -237,16 +242,17 @@ export class MongoDBQueue { public async size(): Promise { return this.col.countDocuments({ - deleted: {$exists: false}, visible: {$lte: now()}, }); } public async inFlight(): Promise { return this.col.countDocuments({ - ack: {$exists: true}, + // For some unknown reason, MongoDB refuses to use the partial index with + // {$exists: true}, but *will* use it if we use {$gt: ''} + // https://www.mongodb.com/community/forums/t/partial-index-is-not-used-during-search/290507/2 + ack: {$gt: ''}, visible: {$gt: now()}, - deleted: {$exists: false}, }); } diff --git a/package.json b/package.json index bab7c97..e561532 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@reedsy/mongodb-queue", - "version": "7.1.1", + "version": "8.0.0", "description": "Message queues which uses MongoDB.", "main": "mongodb-queue.js", "scripts": { From c26194da4d5974cbdafd8ccedc67d31d0584831c Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 11 Feb 2025 15:56:43 +0000 Subject: [PATCH 30/31] =?UTF-8?q?=E2=9C=A8=20Allow=20optional=20TTL=20on?= =?UTF-8?q?=20`deleted`=20field?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that `deleted` is a `Date`, we can add a TTL index to it. This change adds an optional `expireAfterSeconds` option, which is passed through to the `deleted` index options if set to a `number`. --- mongodb-queue.ts | 12 ++++++++++-- package.json | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/mongodb-queue.ts b/mongodb-queue.ts index be44b59..ed38b4a 100644 --- a/mongodb-queue.ts +++ b/mongodb-queue.ts @@ -10,7 +10,7 @@ * **/ -import {Collection, Db, Filter, FindOneAndUpdateOptions, ObjectId, Sort, UpdateFilter, WithId} from 'mongodb'; +import {Collection, CreateIndexesOptions, Db, Filter, FindOneAndUpdateOptions, ObjectId, Sort, UpdateFilter, WithId} from 'mongodb'; function now(): string { return (new Date()).toISOString(); @@ -25,6 +25,7 @@ export type QueueOptions = { delay?: number; deadQueue?: MongoDBQueue; maxRetries?: number; + expireAfterSeconds?: number; }; export type AddOptions = { @@ -64,6 +65,7 @@ export class MongoDBQueue { private readonly delay: number; private readonly maxRetries: number; private readonly deadQueue: MongoDBQueue; + private readonly expireAfterSeconds: number; public constructor(db: Db, name: string, opts: QueueOptions = {}) { if (!db) { @@ -76,6 +78,7 @@ export class MongoDBQueue { this.col = db.collection(name); this.visibility = opts.visibility || 30; this.delay = opts.delay || 0; + this.expireAfterSeconds = opts.expireAfterSeconds; if (opts.deadQueue) { this.deadQueue = opts.deadQueue; @@ -84,10 +87,15 @@ export class MongoDBQueue { } public async createIndexes(): Promise { + const deletedOptions: CreateIndexesOptions = {sparse: true}; + if (typeof this.expireAfterSeconds === 'number') { + deletedOptions.expireAfterSeconds = this.expireAfterSeconds; + } + await Promise.all([ this.col.createIndex({visible: 1}, {sparse: true}), this.col.createIndex({ack: 1}, {unique: true, sparse: true}), - this.col.createIndex({deleted: 1}, {sparse: true}), + this.col.createIndex({deleted: 1}, deletedOptions), // Index for efficient counts on in-flight this.col.createIndex({visible: 1, ack: 1}, { diff --git a/package.json b/package.json index e561532..59e6cce 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@reedsy/mongodb-queue", - "version": "8.0.0", + "version": "8.1.0", "description": "Message queues which uses MongoDB.", "main": "mongodb-queue.js", "scripts": { From 5830ed9ec02be9837ebcda8b6ecff8cb391b05c5 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Fri, 14 Feb 2025 12:23:13 +0000 Subject: [PATCH 31/31] =?UTF-8?q?=E2=9C=A8=20Allow=20resetting=20`ack`=20w?= =?UTF-8?q?hen=20calling=20`ping()`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At the moment, when calling: ```js queue.ping({resetTries: true}) ``` The tries are reset as if the job hasn't been picked up, but the `ack` is left as-is. This isn't necessarily problematic in the operation of the queue, but it does mean that the pinged job will still show up as [in-flight][1]. This change adds an optional `resetAck` flag, which will also unset the `ack`, and means that the job can be marked as not in-flight, as if it has never been picked up. [1]: https://github.com/reedsy/mongodb-queue/blob/6133fc9367f4fce719e36d8866841d531e956b6b/mongodb-queue.ts#L262 --- mongodb-queue.ts | 5 +++++ package.json | 2 +- test/ping.js | 27 +++++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/mongodb-queue.ts b/mongodb-queue.ts index ed38b4a..8cbd6f0 100644 --- a/mongodb-queue.ts +++ b/mongodb-queue.ts @@ -39,6 +39,7 @@ export type GetOptions = { export type PingOptions = { visibility?: number; resetTries?: boolean; + resetAck?: boolean; }; export type BaseMessage = { @@ -205,6 +206,10 @@ export class MongoDBQueue { }; } + if (opts.resetAck) { + update.$unset = {ack: 1}; + } + const msg = await this.col.findOneAndUpdate(query, update, options); if (!msg.value) { throw new Error('Queue.ping(): Unidentified ack : ' + ack); diff --git a/package.json b/package.json index 59e6cce..adf7ba0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@reedsy/mongodb-queue", - "version": "8.1.0", + "version": "8.2.0", "description": "Message queues which uses MongoDB.", "main": "mongodb-queue.js", "scripts": { diff --git a/test/ping.js b/test/ping.js index 54ec2f8..eedca1a 100644 --- a/test/ping.js +++ b/test/ping.js @@ -109,6 +109,33 @@ setup().then(({client, db}) => { t.end(); }); + test('ping: reset ack', async function(t) { + const queue = new MongoDBQueue(db, 'ping', {visibility: 3}); + let msg; + let id; + + id = await queue.add('Hello, World!'); + t.ok(id, 'There is an id returned when adding a message.'); + msg = await queue.get(); + const ack = msg.ack; + // message should reset in three seconds + t.ok(msg.id, 'Got a msg.id (sanity check)'); + await timeout(2000); + id = await queue.ping(msg.ack, {resetAck: true}); + t.ok(id, 'Received an id when acking this message'); + // wait until the msg has returned to the queue + await timeout(6000); + msg = await queue.get(); + t.notEqual(ack, msg.ack, 'Ack was reset'); + await queue.ack(msg.ack); + msg = await queue.get(); + // no more messages + t.ok(!msg, 'No msg received'); + + t.pass('Finished test ok'); + t.end(); + }); + test('client.close()', function(t) { t.pass('client.close()'); client.close();