diff --git a/package-lock.json b/package-lock.json index a1d43c6..c23b092 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "dependencies": { "ioredis": "5.x", "lru-cache": "10.x", + "msgpackr": "1.x", "redlock": "4.x" }, "devDependencies": { @@ -745,6 +746,78 @@ "license": "MIT", "optional": true }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz", + "integrity": "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz", + "integrity": "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz", + "integrity": "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz", + "integrity": "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz", + "integrity": "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz", + "integrity": "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -1806,6 +1879,15 @@ "node": ">=0.10" } }, + "node_modules/detect-libc": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz", + "integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==", + "optional": true, + "engines": { + "node": ">=8" + } + }, "node_modules/diff": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/diff/-/diff-5.2.0.tgz", @@ -3127,6 +3209,35 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", "license": "MIT" }, + "node_modules/msgpackr": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.0.tgz", + "integrity": "sha512-I8qXuuALqJe5laEBYoFykChhSXLikZmUhccjGsPuSJ/7uPip2TJ7lwdIQwWSAi0jGZDXv4WOP8Qg65QZRuXxXw==", + "optionalDependencies": { + "msgpackr-extract": "^3.0.2" + } + }, + "node_modules/msgpackr-extract": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz", + "integrity": "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "node-gyp-build-optional-packages": "5.2.2" + }, + "bin": { + "download-msgpackr-prebuilds": "bin/download-prebuilds.js" + }, + "optionalDependencies": { + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3" + } + }, "node_modules/mute-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-1.0.0.tgz", @@ -3158,6 +3269,20 @@ "path-to-regexp": "^6.2.1" } }, + "node_modules/node-gyp-build-optional-packages": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz", + "integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==", + "optional": true, + "dependencies": { + "detect-libc": "^2.0.1" + }, + "bin": { + "node-gyp-build-optional-packages": "bin.js", + "node-gyp-build-optional-packages-optional": "optional.js", + "node-gyp-build-optional-packages-test": "build-test.js" + } + }, "node_modules/node-preload": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/node-preload/-/node-preload-0.2.1.tgz", diff --git a/package.json b/package.json index 8d5d820..13a6d1d 100644 --- a/package.json +++ b/package.json @@ -72,6 +72,7 @@ }, "dependencies": { "ioredis": "5.x", + "msgpackr": "1.x", "lru-cache": "10.x", "redlock": "4.x" } diff --git a/src/lib/RedisCache.ts b/src/lib/RedisCache.ts index c57f466..ccc0489 100644 --- a/src/lib/RedisCache.ts +++ b/src/lib/RedisCache.ts @@ -1,8 +1,10 @@ import Redis from 'ioredis'; import * as Redlock from 'redlock'; +import { Packr } from 'msgpackr'; import { CachableValue, CacheInstance } from './CacheInstance'; +const MPACK = new Packr({ moreTypes: true }); // `moreTypes: true` to get free support for Set & Map /** * Wrapper class for using Redis as a cache. @@ -21,6 +23,7 @@ export class RedisCache extends CacheInstance { public static TRUE_VALUE = 'f405eed4-507c-4aa5-a6d2-c1813d584b8f-TRUE'; public static FALSE_VALUE = 'f405eed4-507c-4aa5-a6d2-c1813d584b8f-FALSE'; public static JSON_PREFIX = 'f405eed4-507c-4aa5-a6d2-c1813d584b8f-JSON'; + public static MSGP_PREFIX = 'f405eed4-507c-4aa5-a6d2-c1813d584b8f-MSGP'; public static ERROR_PREFIX = 'f405eed4-507c-4aa5-a6d2-c1813d584b8f-ERROR'; public static NUMBER_PREFIX = 'f405eed4-507c-4aa5-a6d2-c1813d584b8f-NUMBER'; @@ -166,19 +169,10 @@ export class RedisCache extends CacheInstance { } if (value instanceof Object) { - return RedisCache.JSON_PREFIX + JSON.stringify(value, (key, value) => { - if (value instanceof Set) { - return { __dataType: 'Set', value: Array.from(value) }; - } else if (value instanceof Map) { - return { __dataType: 'Map', value: Array.from(value) }; - } else { - return value; - } - }); + return `${RedisCache.MSGP_PREFIX}${MPACK.pack(value).toString('binary')}`; } return value; - } /** @@ -212,6 +206,10 @@ export class RedisCache extends CacheInstance { return false; } + if (value.startsWith(RedisCache.MSGP_PREFIX)) { + return MPACK.unpack(Buffer.from(value.substring(RedisCache.MSGP_PREFIX.length), 'binary')); + } + if (value.startsWith(RedisCache.ERROR_PREFIX)) { const deserializedError = JSON.parse(value.substring(RedisCache.ERROR_PREFIX.length)); // return error, restoring potential Error metadata set as object properties @@ -240,7 +238,6 @@ export class RedisCache extends CacheInstance { } return value; - } /** diff --git a/test/RedisCache_test.ts b/test/RedisCache_test.ts index 5867a2f..c9d2062 100644 --- a/test/RedisCache_test.ts +++ b/test/RedisCache_test.ts @@ -53,11 +53,11 @@ describe('RedisCache', () => { }, }; let value = RedisCache.serializeValue(obj); - expect(value.startsWith(RedisCache.JSON_PREFIX)).to.be.true; + expect(value.startsWith(RedisCache.MSGP_PREFIX)).to.be.true; value = RedisCache.deserializeValue(value); expect(value).to.deep.equal(obj); }); - + it('can serialize an object with a nested map', () => { const mapStructure: Map { }, }; let value = RedisCache.serializeValue(obj); - expect(value.startsWith(RedisCache.JSON_PREFIX)).to.be.true; + expect(value.startsWith(RedisCache.MSGP_PREFIX)).to.be.true; value = RedisCache.deserializeValue(value); expect(value).to.deep.equal(obj); }); - + it('can serialize an object with a nested set', () => { const setStructure: Set = new Set(); setStructure.add('key1'); @@ -100,7 +100,7 @@ describe('RedisCache', () => { }, }; let value = RedisCache.serializeValue(obj); - expect(value.startsWith(RedisCache.JSON_PREFIX)).to.be.true; + expect(value.startsWith(RedisCache.MSGP_PREFIX)).to.be.true; value = RedisCache.deserializeValue(value); expect(value).to.deep.equal(obj); }); @@ -222,6 +222,47 @@ describe('RedisCache', () => { expect(await cache.itemCount()).to.equal(1); }); + + it('can set a msgpacked object', async function (): Promise { + if (!process.env.TEST_REDIS_URL) { + this.skip(); + } + + const cache = new RedisCache(process.env.TEST_REDIS_URL as string); + await cache.isReady(); + + // Just to be sure that the cache is really empty... + await cache.clear(); + + const wasSet = await cache.setValue('key', { '🍌': '🥔' }); + expect(wasSet).to.be.true; + + const value = await cache.getValue('key'); + expect(value).to.deep.equal({ '🍌': '🥔' }); + + expect(await cache.itemCount()).to.equal(1); + }); + + it('can get a JSON value', async function (): Promise { + if (!process.env.TEST_REDIS_URL) { + this.skip(); + } + + const cache = new RedisCache(process.env.TEST_REDIS_URL as string); + await cache.isReady(); + + // Just to be sure that the cache is really empty... + await cache.clear(); + + // Manual serialization here to avoid the automatic serialization of the setValue method. + const wasSet = await cache.setValue('key', `${RedisCache.JSON_PREFIX}${JSON.stringify({ '🍌': '🥔' })}`); + expect(wasSet).to.be.true; + + const value = await cache.getValue('key'); + expect(value).to.deep.equal({ '🍌': '🥔' }); + + expect(await cache.itemCount()).to.equal(1); + }); }); describe('itemCount', async () => { @@ -258,7 +299,7 @@ describe('RedisCache', () => { await cache.clear(); await cache.setValue('test1', 'value1'); - + const replicationAcknowledged = await cache.waitForReplication(0, 50); // No replicas so we expect 0. This test basically confirms that waitForReplication doesn't crash. 🤷‍♂️