From d928133594384efb3521a513ddfb9029e680dc7c Mon Sep 17 00:00:00 2001 From: fenos Date: Fri, 8 Dec 2023 13:07:57 +0000 Subject: [PATCH] feat: integrate tus upload as default upload mechanism --- infra/docker-compose.yml | 4 +- infra/kong/kong.yml | 5 + infra/storage/Dockerfile | 2 +- package-lock.json | 372 ++++++++++++++++++++++++++++++--- package.json | 11 +- src/lib/index.ts | 1 + src/lib/tus.ts | 200 ++++++++++++++++++ src/lib/types.ts | 17 ++ src/packages/StorageFileApi.ts | 122 +++++++++-- test/helpers.ts | 44 ++++ test/storageFileApi.test.ts | 158 +++++++++++++- webpack.config.js | 3 + 12 files changed, 873 insertions(+), 66 deletions(-) create mode 100644 src/lib/tus.ts create mode 100644 test/helpers.ts diff --git a/infra/docker-compose.yml b/infra/docker-compose.yml index f8c9971..b3bbf5f 100644 --- a/infra/docker-compose.yml +++ b/infra/docker-compose.yml @@ -30,7 +30,6 @@ services: GLOBAL_S3_BUCKET: supa-storage-testing # name of s3 bucket where you want to store objects PGRST_JWT_SECRET: super-secret-jwt-token-with-at-least-32-characters-long DATABASE_URL: postgres://postgres:postgres@db:5432/postgres - PGOPTIONS: "-c search_path=storage" AWS_ACCESS_KEY_ID: replace-with-your-aws-key AWS_SECRET_ACCESS_KEY: replace-with-your-aws-secret FILE_SIZE_LIMIT: 52428800 @@ -38,7 +37,8 @@ services: FILE_STORAGE_BACKEND_PATH: /tmp/storage ENABLE_IMAGE_TRANSFORMATION: "true" IMGPROXY_URL: http://imgproxy:8080 - DEBUG: "knex:*" + TUS_URL_PATH: /storage/v1/upload/resumable +# DEBUG: "knex:*" volumes: - assets-volume:/tmp/storage healthcheck: diff --git a/infra/kong/kong.yml b/infra/kong/kong.yml index 118003c..c916da9 100644 --- a/infra/kong/kong.yml +++ b/infra/kong/kong.yml @@ -23,6 +23,11 @@ services: - /storage/v1/ plugins: - name: cors + - name: request-transformer + config: + add: + headers: + - 'Forwarded: host=localhost:8000;proto=http' consumers: - username: 'private-key' keyauth_credentials: diff --git a/infra/storage/Dockerfile b/infra/storage/Dockerfile index c14b3d9..c93fb65 100644 --- a/infra/storage/Dockerfile +++ b/infra/storage/Dockerfile @@ -1,3 +1,3 @@ -FROM supabase/storage-api:v0.35.1 +FROM supabase/storage-api:v0.43.12 RUN apk add curl --no-cache \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 215c99a..9c63d6a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ }, "devDependencies": { "@types/jest": "^26.0.13", + "@types/stream-buffers": "^3.0.7", "form-data": "^4.0.0", "genversion": "^3.0.1", "husky": "^4.3.0", @@ -26,9 +27,17 @@ "ts-jest": "^29.0.0", "ts-loader": "^9.4.2", "typedoc": "^0.22.16", - "typescript": "^4.6.3", + "typescript": "^4.9.5", "webpack": "^5.75.0", "webpack-cli": "^5.0.1" + }, + "peerDependencies": { + "tus-js-client": "^3.1.3" + }, + "peerDependenciesMeta": { + "tus-js-client": { + "optional": true + } } }, "node_modules/@ampproject/remapping": { @@ -1345,6 +1354,15 @@ "integrity": "sha512-Hl219/BT5fLAaz6NDkSuhzasy49dwQS/DSdu4MdggFB8zcXv7vflBI3xp7FEmkmdDkBUI2bPUNeMttp2knYdxw==", "dev": true }, + "node_modules/@types/stream-buffers": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@types/stream-buffers/-/stream-buffers-3.0.7.tgz", + "integrity": "sha512-azOCy05sXVXrO+qklf0c/B07H/oHaIuDDAiHPVwlk3A9Ek+ksHyTeMajLZl3r76FxpPpxem//4Te61G1iW3Giw==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/tough-cookie": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.2.tgz", @@ -1914,10 +1932,10 @@ } }, "node_modules/buffer-from": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", - "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==", - "dev": true + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "devOptional": true }, "node_modules/call-bind": { "version": "1.0.2", @@ -2078,6 +2096,17 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "dev": true }, + "node_modules/combine-errors": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/combine-errors/-/combine-errors-3.0.3.tgz", + "integrity": "sha512-C8ikRNRMygCwaTx+Ek3Yr+OuZzgZjduCOfSQBjbM8V3MfgcjSTeto/GXP6PAwKvJz/v15b7GHZvx5rOlczFw/Q==", + "optional": true, + "peer": true, + "dependencies": { + "custom-error-instance": "2.1.1", + "lodash.uniqby": "4.5.0" + } + }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -2168,6 +2197,13 @@ "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==", "dev": true }, + "node_modules/custom-error-instance": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/custom-error-instance/-/custom-error-instance-2.1.1.tgz", + "integrity": "sha512-p6JFxJc3M4OTD2li2qaHkDCw9SfMw82Ldr6OC9Je1aXiGfhx2W8p3GaoeaGrPJTUN9NirTM/KTxHWMUdR1rsUg==", + "optional": true, + "peer": true + }, "node_modules/data-urls": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.2.tgz", @@ -2819,7 +2855,7 @@ "version": "4.2.10", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==", - "dev": true + "devOptional": true }, "node_modules/has": { "version": "1.0.3", @@ -3207,7 +3243,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz", "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==", - "dev": true, + "devOptional": true, "engines": { "node": ">=8" } @@ -4407,6 +4443,13 @@ "url": "https://github.com/chalk/supports-color?sponsor=1" } }, + "node_modules/js-base64": { + "version": "3.7.5", + "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.7.5.tgz", + "integrity": "sha512-3MEt5DTINKqfScXKfJFrRbxkrnk2AxPWGBL/ycjz4dK8iqiSJ06UxD8jh8xuh6p10TX4t2+7FsBYVxxQbMg+qA==", + "optional": true, + "peer": true + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -4623,12 +4666,82 @@ "node": ">=8" } }, + "node_modules/lodash._baseiteratee": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/lodash._baseiteratee/-/lodash._baseiteratee-4.7.0.tgz", + "integrity": "sha512-nqB9M+wITz0BX/Q2xg6fQ8mLkyfF7MU7eE+MNBNjTHFKeKaZAPEzEg+E8LWxKWf1DQVflNEn9N49yAuqKh2mWQ==", + "optional": true, + "peer": true, + "dependencies": { + "lodash._stringtopath": "~4.8.0" + } + }, + "node_modules/lodash._basetostring": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/lodash._basetostring/-/lodash._basetostring-4.12.0.tgz", + "integrity": "sha512-SwcRIbyxnN6CFEEK4K1y+zuApvWdpQdBHM/swxP962s8HIxPO3alBH5t3m/dl+f4CMUug6sJb7Pww8d13/9WSw==", + "optional": true, + "peer": true + }, + "node_modules/lodash._baseuniq": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash._baseuniq/-/lodash._baseuniq-4.6.0.tgz", + "integrity": "sha512-Ja1YevpHZctlI5beLA7oc5KNDhGcPixFhcqSiORHNsp/1QTv7amAXzw+gu4YOvErqVlMVyIJGgtzeepCnnur0A==", + "optional": true, + "peer": true, + "dependencies": { + "lodash._createset": "~4.0.0", + "lodash._root": "~3.0.0" + } + }, + "node_modules/lodash._createset": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/lodash._createset/-/lodash._createset-4.0.3.tgz", + "integrity": "sha512-GTkC6YMprrJZCYU3zcqZj+jkXkrXzq3IPBcF/fIPpNEAB4hZEtXU8zp/RwKOvZl43NUmwDbyRk3+ZTbeRdEBXA==", + "optional": true, + "peer": true + }, + "node_modules/lodash._root": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/lodash._root/-/lodash._root-3.0.1.tgz", + "integrity": "sha512-O0pWuFSK6x4EXhM1dhZ8gchNtG7JMqBtrHdoUFUWXD7dJnNSUze1GuyQr5sOs0aCvgGeI3o/OJW8f4ca7FDxmQ==", + "optional": true, + "peer": true + }, + "node_modules/lodash._stringtopath": { + "version": "4.8.0", + "resolved": "https://registry.npmjs.org/lodash._stringtopath/-/lodash._stringtopath-4.8.0.tgz", + "integrity": "sha512-SXL66C731p0xPDC5LZg4wI5H+dJo/EO4KTqOMwLYCH3+FmmfAKJEZCm6ohGpI+T1xwsDsJCfL4OnhorllvlTPQ==", + "optional": true, + "peer": true, + "dependencies": { + "lodash._basetostring": "~4.12.0" + } + }, "node_modules/lodash.memoize": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", "integrity": "sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag==", "dev": true }, + "node_modules/lodash.throttle": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.throttle/-/lodash.throttle-4.1.1.tgz", + "integrity": "sha512-wIkUCfVKpVsWo3JSZlc+8MB5it+2AN5W8J7YVMST30UrvcQNZ1Okbj+rbVniijTWE6FGYy4XJq/rHkas8qJMLQ==", + "optional": true, + "peer": true + }, + "node_modules/lodash.uniqby": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.uniqby/-/lodash.uniqby-4.5.0.tgz", + "integrity": "sha512-IRt7cfTtHy6f1aRVA5n7kT8rgN3N1nH6MOWLcHfpWG2SH19E3JksLK38MktLxZDhlAjCP9jpIXkOnRXlu6oByQ==", + "optional": true, + "peer": true, + "dependencies": { + "lodash._baseiteratee": "~4.7.0", + "lodash._baseuniq": "~4.6.0" + } + }, "node_modules/lru-cache": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", @@ -5495,6 +5608,18 @@ "node": ">= 6" } }, + "node_modules/proper-lockfile": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz", + "integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==", + "optional": true, + "peer": true, + "dependencies": { + "graceful-fs": "^4.2.4", + "retry": "^0.12.0", + "signal-exit": "^3.0.2" + } + }, "node_modules/psl": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz", @@ -5540,7 +5665,7 @@ "version": "2.2.0", "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==", - "dev": true + "devOptional": true }, "node_modules/randombytes": { "version": "2.1.0", @@ -5608,7 +5733,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", - "dev": true + "devOptional": true }, "node_modules/resolve": { "version": "1.20.0", @@ -5662,6 +5787,16 @@ "node": ">=10" } }, + "node_modules/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "optional": true, + "peer": true, + "engines": { + "node": ">= 4" + } + }, "node_modules/rimraf": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", @@ -5678,10 +5813,24 @@ } }, "node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", - "dev": true + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] }, "node_modules/safer-buffer": { "version": "2.1.2", @@ -5822,7 +5971,7 @@ "version": "3.0.7", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", - "dev": true + "devOptional": true }, "node_modules/sisteransi": { "version": "1.0.5", @@ -6288,6 +6437,22 @@ "node": ">=10" } }, + "node_modules/tus-js-client": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/tus-js-client/-/tus-js-client-3.1.3.tgz", + "integrity": "sha512-n9k6rI/nPOuP2TaqPG6Ogz3a3V1cSH9en7N0VH4gh95jmG8JA58TJzLms2lBfb7aKVb3fdUunqYEG3WnQnZRvQ==", + "optional": true, + "peer": true, + "dependencies": { + "buffer-from": "^1.1.2", + "combine-errors": "^3.0.3", + "is-stream": "^2.0.0", + "js-base64": "^3.7.2", + "lodash.throttle": "^4.1.1", + "proper-lockfile": "^4.1.2", + "url-parse": "^1.5.7" + } + }, "node_modules/type-check": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", @@ -6384,9 +6549,9 @@ } }, "node_modules/typescript": { - "version": "4.6.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.6.3.tgz", - "integrity": "sha512-yNIatDa5iaofVozS/uQJEl3JRWLKKGJKh6Yaiv0GLGSuhpFJe7P3SbHZ8/yjAHRQwKRoA6YZqlfjXWmVzoVSMw==", + "version": "4.9.5", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz", + "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==", "dev": true, "bin": { "tsc": "bin/tsc", @@ -6459,7 +6624,7 @@ "version": "1.5.10", "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", - "dev": true, + "devOptional": true, "dependencies": { "querystringify": "^2.1.1", "requires-port": "^1.0.0" @@ -7966,6 +8131,15 @@ "integrity": "sha512-Hl219/BT5fLAaz6NDkSuhzasy49dwQS/DSdu4MdggFB8zcXv7vflBI3xp7FEmkmdDkBUI2bPUNeMttp2knYdxw==", "dev": true }, + "@types/stream-buffers": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@types/stream-buffers/-/stream-buffers-3.0.7.tgz", + "integrity": "sha512-azOCy05sXVXrO+qklf0c/B07H/oHaIuDDAiHPVwlk3A9Ek+ksHyTeMajLZl3r76FxpPpxem//4Te61G1iW3Giw==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/tough-cookie": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.2.tgz", @@ -8422,10 +8596,10 @@ } }, "buffer-from": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", - "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==", - "dev": true + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "devOptional": true }, "call-bind": { "version": "1.0.2", @@ -8538,6 +8712,17 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "dev": true }, + "combine-errors": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/combine-errors/-/combine-errors-3.0.3.tgz", + "integrity": "sha512-C8ikRNRMygCwaTx+Ek3Yr+OuZzgZjduCOfSQBjbM8V3MfgcjSTeto/GXP6PAwKvJz/v15b7GHZvx5rOlczFw/Q==", + "optional": true, + "peer": true, + "requires": { + "custom-error-instance": "2.1.1", + "lodash.uniqby": "4.5.0" + } + }, "combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -8618,6 +8803,13 @@ } } }, + "custom-error-instance": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/custom-error-instance/-/custom-error-instance-2.1.1.tgz", + "integrity": "sha512-p6JFxJc3M4OTD2li2qaHkDCw9SfMw82Ldr6OC9Je1aXiGfhx2W8p3GaoeaGrPJTUN9NirTM/KTxHWMUdR1rsUg==", + "optional": true, + "peer": true + }, "data-urls": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.2.tgz", @@ -9103,7 +9295,7 @@ "version": "4.2.10", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==", - "dev": true + "devOptional": true }, "has": { "version": "1.0.3", @@ -9373,7 +9565,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz", "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==", - "dev": true + "devOptional": true }, "is-string": { "version": "1.0.5", @@ -10261,6 +10453,13 @@ } } }, + "js-base64": { + "version": "3.7.5", + "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.7.5.tgz", + "integrity": "sha512-3MEt5DTINKqfScXKfJFrRbxkrnk2AxPWGBL/ycjz4dK8iqiSJ06UxD8jh8xuh6p10TX4t2+7FsBYVxxQbMg+qA==", + "optional": true, + "peer": true + }, "js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -10426,12 +10625,82 @@ "p-locate": "^4.1.0" } }, + "lodash._baseiteratee": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/lodash._baseiteratee/-/lodash._baseiteratee-4.7.0.tgz", + "integrity": "sha512-nqB9M+wITz0BX/Q2xg6fQ8mLkyfF7MU7eE+MNBNjTHFKeKaZAPEzEg+E8LWxKWf1DQVflNEn9N49yAuqKh2mWQ==", + "optional": true, + "peer": true, + "requires": { + "lodash._stringtopath": "~4.8.0" + } + }, + "lodash._basetostring": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/lodash._basetostring/-/lodash._basetostring-4.12.0.tgz", + "integrity": "sha512-SwcRIbyxnN6CFEEK4K1y+zuApvWdpQdBHM/swxP962s8HIxPO3alBH5t3m/dl+f4CMUug6sJb7Pww8d13/9WSw==", + "optional": true, + "peer": true + }, + "lodash._baseuniq": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash._baseuniq/-/lodash._baseuniq-4.6.0.tgz", + "integrity": "sha512-Ja1YevpHZctlI5beLA7oc5KNDhGcPixFhcqSiORHNsp/1QTv7amAXzw+gu4YOvErqVlMVyIJGgtzeepCnnur0A==", + "optional": true, + "peer": true, + "requires": { + "lodash._createset": "~4.0.0", + "lodash._root": "~3.0.0" + } + }, + "lodash._createset": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/lodash._createset/-/lodash._createset-4.0.3.tgz", + "integrity": "sha512-GTkC6YMprrJZCYU3zcqZj+jkXkrXzq3IPBcF/fIPpNEAB4hZEtXU8zp/RwKOvZl43NUmwDbyRk3+ZTbeRdEBXA==", + "optional": true, + "peer": true + }, + "lodash._root": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/lodash._root/-/lodash._root-3.0.1.tgz", + "integrity": "sha512-O0pWuFSK6x4EXhM1dhZ8gchNtG7JMqBtrHdoUFUWXD7dJnNSUze1GuyQr5sOs0aCvgGeI3o/OJW8f4ca7FDxmQ==", + "optional": true, + "peer": true + }, + "lodash._stringtopath": { + "version": "4.8.0", + "resolved": "https://registry.npmjs.org/lodash._stringtopath/-/lodash._stringtopath-4.8.0.tgz", + "integrity": "sha512-SXL66C731p0xPDC5LZg4wI5H+dJo/EO4KTqOMwLYCH3+FmmfAKJEZCm6ohGpI+T1xwsDsJCfL4OnhorllvlTPQ==", + "optional": true, + "peer": true, + "requires": { + "lodash._basetostring": "~4.12.0" + } + }, "lodash.memoize": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", "integrity": "sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag==", "dev": true }, + "lodash.throttle": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.throttle/-/lodash.throttle-4.1.1.tgz", + "integrity": "sha512-wIkUCfVKpVsWo3JSZlc+8MB5it+2AN5W8J7YVMST30UrvcQNZ1Okbj+rbVniijTWE6FGYy4XJq/rHkas8qJMLQ==", + "optional": true, + "peer": true + }, + "lodash.uniqby": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.uniqby/-/lodash.uniqby-4.5.0.tgz", + "integrity": "sha512-IRt7cfTtHy6f1aRVA5n7kT8rgN3N1nH6MOWLcHfpWG2SH19E3JksLK38MktLxZDhlAjCP9jpIXkOnRXlu6oByQ==", + "optional": true, + "peer": true, + "requires": { + "lodash._baseiteratee": "~4.7.0", + "lodash._baseuniq": "~4.6.0" + } + }, "lru-cache": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", @@ -11081,6 +11350,18 @@ "sisteransi": "^1.0.5" } }, + "proper-lockfile": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz", + "integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==", + "optional": true, + "peer": true, + "requires": { + "graceful-fs": "^4.2.4", + "retry": "^0.12.0", + "signal-exit": "^3.0.2" + } + }, "psl": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz", @@ -11113,7 +11394,7 @@ "version": "2.2.0", "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==", - "dev": true + "devOptional": true }, "randombytes": { "version": "2.1.0", @@ -11171,7 +11452,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", - "dev": true + "devOptional": true }, "resolve": { "version": "1.20.0", @@ -11212,6 +11493,13 @@ "integrity": "sha512-X2UW6Nw3n/aMgDVy+0rSqgHlv39WZAlZrXCdnbyEiKm17DSqHX4MmQMaST3FbeWR5FTuRcUwYAziZajji0Y7mg==", "dev": true }, + "retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "optional": true, + "peer": true + }, "rimraf": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", @@ -11222,9 +11510,9 @@ } }, "safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", "dev": true }, "safer-buffer": { @@ -11335,7 +11623,7 @@ "version": "3.0.7", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", - "dev": true + "devOptional": true }, "sisteransi": { "version": "1.0.5", @@ -11662,6 +11950,22 @@ } } }, + "tus-js-client": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/tus-js-client/-/tus-js-client-3.1.3.tgz", + "integrity": "sha512-n9k6rI/nPOuP2TaqPG6Ogz3a3V1cSH9en7N0VH4gh95jmG8JA58TJzLms2lBfb7aKVb3fdUunqYEG3WnQnZRvQ==", + "optional": true, + "peer": true, + "requires": { + "buffer-from": "^1.1.2", + "combine-errors": "^3.0.3", + "is-stream": "^2.0.0", + "js-base64": "^3.7.2", + "lodash.throttle": "^4.1.1", + "proper-lockfile": "^4.1.2", + "url-parse": "^1.5.7" + } + }, "type-check": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", @@ -11730,9 +12034,9 @@ } }, "typescript": { - "version": "4.6.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.6.3.tgz", - "integrity": "sha512-yNIatDa5iaofVozS/uQJEl3JRWLKKGJKh6Yaiv0GLGSuhpFJe7P3SbHZ8/yjAHRQwKRoA6YZqlfjXWmVzoVSMw==", + "version": "4.9.5", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz", + "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==", "dev": true }, "unbox-primitive": { @@ -11776,7 +12080,7 @@ "version": "1.5.10", "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", - "dev": true, + "devOptional": true, "requires": { "querystringify": "^2.1.1", "requires-port": "^1.0.0" diff --git a/package.json b/package.json index 515e796..afca4c4 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ }, "devDependencies": { "@types/jest": "^26.0.13", + "@types/stream-buffers": "^3.0.7", "form-data": "^4.0.0", "genversion": "^3.0.1", "husky": "^4.3.0", @@ -53,10 +54,18 @@ "ts-jest": "^29.0.0", "ts-loader": "^9.4.2", "typedoc": "^0.22.16", - "typescript": "^4.6.3", + "typescript": "^4.9.5", "webpack": "^5.75.0", "webpack-cli": "^5.0.1" }, + "peerDependencies": { + "tus-js-client": "^3.1.3" + }, + "peerDependenciesMeta": { + "tus-js-client": { + "optional": true + } + }, "husky": { "hooks": { "pre-commit": "pretty-quick --staged" diff --git a/src/lib/index.ts b/src/lib/index.ts index fb0c997..2bc9b43 100644 --- a/src/lib/index.ts +++ b/src/lib/index.ts @@ -2,3 +2,4 @@ export * from '../packages/StorageBucketApi' export * from '../packages/StorageFileApi' export * from './types' export * from './constants' +export * from './tus' diff --git a/src/lib/tus.ts b/src/lib/tus.ts new file mode 100644 index 0000000..e8d01eb --- /dev/null +++ b/src/lib/tus.ts @@ -0,0 +1,200 @@ +import { StorageApiError, StorageError } from './errors' +import { FileBody } from './types' + +// @ts-ignore -- potentially the library is not installed +import type { Upload, DetailedError, PreviousUpload } from 'tus-js-client' + +export function getTusJsPeerDep() { + try { + // @ts-ignore potentially the library is not installed + const tusJS = require('tus-js-client') + // @ts-ignore potentially the library is not installed + return tusJS as Awaited + } catch (e) { + return + } +} + +export function isTusJSAvailable() { + return Boolean(getTusJsPeerDep()) +} + +export type TusUpload = any extends Upload ? never : Upload +export type TusUploadOptionFallback = TusUpload extends never + ? Fallback + : Omit + +export interface TusUploadOptions { + upsert?: boolean + contentType?: string + onProgress?: (percentage: number, bytesUploaded: number, bytesTotal: number) => void + onSuccess: (response: SuccessResponse) => void + onError?: (response: ErrorResponse) => void + cacheControl?: number + authorization?: string + formDataFileKey?: string + tusOptions?: Pick< + TusUpload['options'], + | 'retryDelays' + | 'onChunkComplete' + | 'onShouldRetry' + | 'onUploadUrlAvailable' + | 'overridePatchMethod' + | 'storeFingerprintForResuming' + | 'removeFingerprintOnSuccess' + | 'uploadDataDuringCreation' + | 'urlStorage' + | 'fileReader' + | 'httpStack' + | 'uploadSize' + | 'fingerprint' + | 'onAfterResponse' + | 'onBeforeRequest' + > +} + +type SuccessResponse = { + data: { path: string } + error: null +} + +type ErrorResponse = { + error: StorageError + data: null +} + +type Response = SuccessResponse | ErrorResponse + +export class TerminateError extends Error {} + +export class TusUploader { + uploadResource: Upload + inProgress = false + + constructor( + private readonly url: string, + private readonly bucketId: string, + private readonly path: string, + private readonly file: FileBody, + private readonly options: TusUploadOptions + ) { + this.uploadResource = this.prepareUpload() + } + + async abort() { + await this.uploadResource.abort(true) + this.inProgress = false + } + + async pause() { + await this.uploadResource.abort(false) + this.inProgress = false + } + + listPreviousUploads() { + return this.uploadResource.findPreviousUploads() + } + + async clearAllPreviousUploads() { + const tusJs = getTusJsPeerDep() + if (!tusJs) { + throw new Error('tus-js-client not installed install with: npm install tus-js-client') + } + + const previousUploads = await this.uploadResource.findPreviousUploads() + return Promise.all( + previousUploads.map((upload: PreviousUpload) => + tusJs.Upload.terminate((upload as any).uploadUrl) + ) + ) + } + + async startOrResume(index = 0) { + const previousUploads = await this.uploadResource.findPreviousUploads() + + if (previousUploads.length) { + this.uploadResource.resumeFromPreviousUpload(previousUploads[index]) + } + return this.start() + } + + start() { + if (this.inProgress) { + console.warn('an upload is already in progress for this resource') + return + } + + this.inProgress = true + this.uploadResource.start() + } + + protected prepareUpload() { + const tusJs = getTusJsPeerDep() + if (!tusJs) { + throw new Error('tus-js-client not installed install with: npm install tus-js-client') + } + + const options = this.options + const file = this.file + const url = this.url + const bucketId = this.bucketId + const path = this.path + + const headers: Record = {} + let fileBody = this.file as File | Blob | Pick + let contentType: string = '' + let cacheControl: string = '' + + if (options?.authorization) { + headers.authorization = options.authorization + } + + if (options?.upsert) { + headers['x-upsert'] = 'true' + } + + if (file instanceof FormData) { + fileBody = file.get(options?.formDataFileKey || 'file') as Blob + contentType = (file.get('contentType') as string) || '' + cacheControl = (file.get('cacheControl') as string) || '' + } + + return new tusJs.Upload(fileBody, { + endpoint: `${url}/upload/resumable`, + retryDelays: [0, 200, 500, 1000, 2000], + removeFingerprintOnSuccess: true, + storeFingerprintForResuming: true, + headers: headers, + chunkSize: 6 * 1024 * 1024, + metadata: { + bucketName: bucketId, + objectName: path, + contentType: contentType || options?.contentType || 'text/plain;charset=UTF-8', + cacheControl: cacheControl || options?.cacheControl?.toString() || '3600', + }, + onError: (error: DetailedError | Error) => { + this.inProgress = false + if ('originalResponse' in error) { + options?.onError?.({ + data: null, + error: new StorageApiError( + error.originalResponse?.getBody() || error.message, + error.originalResponse?.getStatus() || 500 + ), + }) + } else { + options?.onError?.({ data: null, error: new StorageApiError(error.message, 500) }) + } + }, + onProgress: (bytesUploaded: number, bytesTotal: number) => { + const percentage = (bytesUploaded / bytesTotal) * 100 + options?.onProgress?.(percentage, bytesUploaded, bytesTotal) + }, + onSuccess: () => { + this.inProgress = false + options.onSuccess({ data: { path: path }, error: null }) + }, + ...(options?.tusOptions || {}), + }) + } +} diff --git a/src/lib/types.ts b/src/lib/types.ts index fbccbb9..7be1876 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -43,6 +43,11 @@ export interface FileOptions { * The duplex option is a string parameter that enables or disables duplex streaming, allowing for both reading and writing data in the same stream. It can be passed as an option to the fetch() method. */ duplex?: string + + /** + * The signal option allows you to communicate with a DOM request (such as a Fetch) and abort it if required via an AbortController object. + */ + signal?: AbortSignal } export interface SearchOptions { @@ -109,3 +114,15 @@ export interface TransformOptions { */ format?: 'origin' } + +export type FileBody = + | ArrayBuffer + | ArrayBufferView + | Blob + | Buffer + | File + | FormData + | NodeJS.ReadableStream + | ReadableStream + | URLSearchParams + | string diff --git a/src/packages/StorageFileApi.ts b/src/packages/StorageFileApi.ts index fb17eef..5565951 100644 --- a/src/packages/StorageFileApi.ts +++ b/src/packages/StorageFileApi.ts @@ -7,7 +7,12 @@ import { SearchOptions, FetchParameters, TransformOptions, -} from '../lib/types' + FileBody, + TusUploader, + TusUploadOptions, + TusUploadOptionFallback, + isTusJSAvailable, +} from '../lib' const DEFAULT_SEARCH_OPTIONS = { limit: 100, @@ -24,17 +29,15 @@ const DEFAULT_FILE_OPTIONS: FileOptions = { upsert: false, } -type FileBody = - | ArrayBuffer - | ArrayBufferView - | Blob - | Buffer - | File - | FormData - | NodeJS.ReadableStream - | ReadableStream - | URLSearchParams - | string +export type UploadOptions< + T extends TusUploadOptionFallback = TusUploadOptionFallback +> = T extends FileOptions + ? FileOptions + : (FileOptions & { forceStandardUpload: true }) | (T & { forceStandardUpload?: false }) + +interface WithAbortSignal { + signal?: AbortSignal +} export default class StorageFileApi { protected url: string @@ -45,7 +48,7 @@ export default class StorageFileApi { constructor( url: string, headers: { [key: string]: string } = {}, - bucketId?: string, + bucketId: string, fetch?: Fetch ) { this.url = url @@ -68,7 +71,7 @@ export default class StorageFileApi { fileOptions?: FileOptions ): Promise< | { - data: { id: string, path: string, fullPath: string } + data: { id: string; path: string; fullPath: string } error: null } | { @@ -103,6 +106,7 @@ export default class StorageFileApi { method, body: body as BodyInit, headers, + signal: fileOptions?.signal, ...(options?.duplex ? { duplex: options.duplex } : {}), }) @@ -126,16 +130,29 @@ export default class StorageFileApi { } } + createResumableUpload( + path: string, + fileBody: FileBody, + fileOptions?: Omit + ) { + const authorizationHeader = this.headers?.authorization || this.headers?.Authorization + return new TusUploader(this.url, this.bucketId as string, path, fileBody, { + authorization: authorizationHeader, + ...fileOptions, + } as TusUploadOptions) + } + /** * Uploads a file to an existing bucket. * * @param path The file path, including the file name. Should be of the format `folder/subfolder/filename.png`. The bucket must already exist before attempting to upload. * @param fileBody The body of the file to be stored in the bucket. + * @param fileOptions */ async upload( path: string, fileBody: FileBody, - fileOptions?: FileOptions + fileOptions?: UploadOptions & WithAbortSignal ): Promise< | { data: { path: string } @@ -146,7 +163,41 @@ export default class StorageFileApi { error: StorageError } > { - return this.uploadOrUpdate('POST', path, fileBody, fileOptions) + const shouldUseTUS = isTusJSAvailable() + + if (shouldUseTUS && !fileOptions?.forceStandardUpload) { + return new Promise((resolve, reject) => { + const abortHandler = () => { + upload.abort() + reject(new StorageError('Upload Aborted')) + } + + const upload = this.createResumableUpload(path, fileBody, { + ...(fileOptions as TusUploadOptions), + onSuccess: (...args) => { + fileOptions?.signal?.removeEventListener('abort', abortHandler) + resolve(...args) + }, + onError: (...args) => { + fileOptions?.signal?.removeEventListener('abort', abortHandler) + resolve(...args) + }, + }) + + fileOptions?.signal?.addEventListener('abort', () => { + if (fileOptions?.signal?.reason) { + upload.abort() + } else { + upload.pause() + } + reject(new StorageError('Upload Aborted')) + }) + + upload.startOrResume() + }) + } + + return this.uploadOrUpdate('POST', path, fileBody, fileOptions as FileOptions) } /** @@ -279,7 +330,7 @@ export default class StorageFileApi { | ReadableStream | URLSearchParams | string, - fileOptions?: FileOptions + fileOptions?: UploadOptions & WithAbortSignal ): Promise< | { data: { path: string } @@ -290,7 +341,42 @@ export default class StorageFileApi { error: StorageError } > { - return this.uploadOrUpdate('PUT', path, fileBody, fileOptions) + const shouldUseTus = isTusJSAvailable() + + if (shouldUseTus && !fileOptions?.forceStandardUpload) { + return new Promise((resolve, reject) => { + const abortHandler = () => { + upload.abort() + reject(new StorageError('Upload Aborted')) + } + + const upload = this.createResumableUpload(path, fileBody, { + ...(fileOptions as TusUploadOptions), + upsert: true, + onSuccess: (...args) => { + fileOptions?.signal?.removeEventListener('abort', abortHandler) + resolve(...args) + }, + onError: (...args) => { + fileOptions?.signal?.removeEventListener('abort', abortHandler) + resolve(...args) + }, + }) + + fileOptions?.signal?.addEventListener('abort', () => { + if (fileOptions?.signal?.reason) { + upload.abort() + } else { + upload.pause() + } + reject(new StorageError('Upload Aborted')) + }) + + upload.startOrResume() + }) + } + + return this.uploadOrUpdate('PUT', path, fileBody, fileOptions as FileOptions) } /** diff --git a/test/helpers.ts b/test/helpers.ts new file mode 100644 index 0000000..cd49db7 --- /dev/null +++ b/test/helpers.ts @@ -0,0 +1,44 @@ +import { Readable } from 'stream' + +export class SlowReadableStream extends Readable { + private buffer: Buffer + private chunkSize: number + private delay: number + private position: number + private isFirstChunk: boolean + private scheduledRead: boolean + + constructor(buffer: Buffer, chunkSize: number, delay: number) { + super() + this.buffer = buffer + this.chunkSize = chunkSize + this.delay = delay + this.position = 0 + this.isFirstChunk = true + this.scheduledRead = false + } + + _read(): void { + if (this.isFirstChunk) { + this.pushChunk() + this.isFirstChunk = false + } else if (!this.scheduledRead) { + this.scheduledRead = true + setTimeout(() => { + this.scheduledRead = false + this.pushChunk() + }, this.delay) + } + } + + private pushChunk(): void { + if (this.position >= this.buffer.length) { + this.push(null) // Signal end of stream + return + } + + const chunk = this.buffer.slice(this.position, this.position + this.chunkSize) + this.push(chunk) + this.position += this.chunkSize + } +} diff --git a/test/storageFileApi.test.ts b/test/storageFileApi.test.ts index cfa8c43..a556bca 100644 --- a/test/storageFileApi.test.ts +++ b/test/storageFileApi.test.ts @@ -1,11 +1,15 @@ -import { StorageClient } from '../src/index' +import { StorageApiError, StorageClient } from '../src/index' import * as fsp from 'fs/promises' import * as fs from 'fs' import * as path from 'path' import FormData from 'form-data' import assert from 'assert' // @ts-ignore +import { FileUrlStorage } from 'tus-js-client' +// @ts-ignore import fetch from '@supabase/node-fetch' +// @ts-ignore +import { SlowReadableStream } from './helpers' // TODO: need to setup storage-api server for this test const URL = 'http://localhost:8000/storage/v1' @@ -114,32 +118,40 @@ describe('Object API', () => { }) }) - describe('Upload files', () => { + describe('Upload files Standard', () => { test('uploading using form-data', async () => { const bucketName = await newBucket() const formData = new FormData() formData.append('file', file) - const res = await storage.from(bucketName).upload(uploadPath, formData) + const res = await storage.from(bucketName).upload(uploadPath, formData, { + forceStandardUpload: true, + }) expect(res.error).toBeNull() expect(res.data?.path).toEqual(uploadPath) }) test('uploading using buffer', async () => { - const res = await storage.from(bucketName).upload(uploadPath, file) + const res = await storage.from(bucketName).upload(uploadPath, file, { + forceStandardUpload: true, + }) expect(res.error).toBeNull() expect(res.data?.path).toEqual(uploadPath) }) test('uploading using array buffer', async () => { - const res = await storage.from(bucketName).upload(uploadPath, file.buffer) + const res = await storage.from(bucketName).upload(uploadPath, Buffer.from(file.buffer), { + forceStandardUpload: true, + }) expect(res.error).toBeNull() expect(res.data?.path).toEqual(uploadPath) }) test('uploading using blob', async () => { const fileBlob = new Blob([file]) - const res = await storage.from(bucketName).upload(uploadPath, fileBlob) + const res = await storage.from(bucketName).upload(uploadPath, fileBlob, { + forceStandardUpload: true, + }) expect(res.error).toBeNull() expect(res.data?.path).toEqual(uploadPath) }) @@ -147,7 +159,9 @@ describe('Object API', () => { test('uploading using readable stream', async () => { const file = await fs.createReadStream(uploadFilePath('file.txt')) - const res = await storage.from(bucketName).upload(uploadPath, file) + const res = await storage.from(bucketName).upload(uploadPath, file, { + forceStandardUpload: true, + }) expect(res.error).toBeNull() expect(res.data?.path).toEqual(uploadPath) }) @@ -158,7 +172,9 @@ describe('Object API', () => { const res = await storage.from(bucketName).upload(uploadPath, file) expect(res.error).toBeNull() - const updateRes = await storage.from(bucketName).update(uploadPath, file2) + const updateRes = await storage.from(bucketName).update(uploadPath, file2, { + forceStandardUpload: true, + }) expect(updateRes.error).toBeNull() expect(updateRes.data?.path).toEqual(uploadPath) }) @@ -170,7 +186,9 @@ describe('Object API', () => { fileSizeLimit: '1mb', }) - const res = await storage.from(bucketName).upload(uploadPath, file) + const res = await storage.from(bucketName).upload(uploadPath, file, { + forceStandardUpload: true, + }) expect(res.error).toBeNull() }) @@ -181,7 +199,9 @@ describe('Object API', () => { fileSizeLimit: '1kb', }) - const res = await storage.from(bucketName).upload(uploadPath, file) + const res = await storage.from(bucketName).upload(uploadPath, file, { + forceStandardUpload: true, + }) expect(res.error).toEqual({ error: 'Payload too large', message: 'The object exceeded the maximum allowed size', @@ -198,6 +218,7 @@ describe('Object API', () => { const res = await storage.from(bucketName).upload(uploadPath, file, { contentType: 'image/png', + forceStandardUpload: true, }) expect(res.error).toBeNull() }) @@ -210,6 +231,7 @@ describe('Object API', () => { }) const res = await storage.from(bucketName).upload(uploadPath, file, { + forceStandardUpload: true, contentType: 'image/jpeg', }) expect(res.error).toEqual({ @@ -266,6 +288,122 @@ describe('Object API', () => { }) }) + describe('Upload files TUS', () => { + test('upload and update file', async () => { + const file2 = await fsp.readFile(uploadFilePath('file-2.txt')) + + const res = await storage.from(bucketName).upload(uploadPath, file) + expect(res.error).toBeNull() + + const updateRes = await storage.from(bucketName).update(uploadPath, file2) + expect(updateRes.error).toBeNull() + expect(updateRes.data?.path).toEqual(uploadPath) + }) + + test('can upload a file within the file size limit', async () => { + const bucketName = 'with-limit' + Date.now() + await storage.createBucket(bucketName, { + public: true, + fileSizeLimit: '1mb', + }) + + const res = await storage.from(bucketName).upload(uploadPath, file) + expect(res.error).toBeNull() + }) + + test('cannot upload a file that exceed the file size limit', async () => { + const bucketName = 'with-limit' + Date.now() + await storage.createBucket(bucketName, { + public: true, + fileSizeLimit: '1kb', + }) + + const res = await storage.from(bucketName).upload(uploadPath, file) + const err = res.error as StorageApiError + expect(err.status).toEqual(413) + expect(err.message).toEqual('Request Entity Too Large\n') + }) + + test('can pause and resume a file', async () => { + const fileContent = Buffer.alloc(1024 * 1024 * 12) + const uploadPath = `testpath/file-${Date.now()}.txt` + + let fileBody = new SlowReadableStream(fileContent, 1024 * 1024 * 2, 500) + + const successSpy = jest.fn() + const errorSpy = jest.fn() + + let resolveFn: () => void = () => {} + let rejectFn: (err: any) => void = () => {} + + const uploadPromise = new Promise((resolve, reject) => { + resolveFn = resolve + rejectFn = reject + }) + + const upload = storage.from(bucketName).createResumableUpload(uploadPath, fileBody, { + tusOptions: { + urlStorage: new FileUrlStorage(__dirname + '/.tus/fingerprints.info'), + fingerprint: async (file, options) => { + return options?.metadata?.objectName || '' + }, + uploadSize: fileContent.length, + }, + onSuccess: () => { + successSpy() + resolveFn() + }, + onError: (err) => { + errorSpy() + rejectFn(err) + }, + }) + + upload.start() + + await new Promise((resolve) => setTimeout(resolve, 2100)) + await upload?.pause() + await new Promise((resolve) => setTimeout(resolve, 100)) + + upload?.start() + + await uploadPromise + + expect(successSpy).toBeCalledTimes(1) + }, 10000) + + test('can upload a file with a valid mime type', async () => { + const bucketName = 'with-limit' + Date.now() + await storage.createBucket(bucketName, { + public: true, + allowedMimeTypes: ['image/png'], + }) + + const res = await storage.from(bucketName).upload(uploadPath, file, { + contentType: 'image/png', + }) + expect(res.error).toBeNull() + }) + + test('cannot upload a file an invalid mime type', async () => { + const bucketName = 'with-limit' + Date.now() + await storage.createBucket(bucketName, { + public: true, + allowedMimeTypes: ['image/png'], + }) + + const res = await storage.from(bucketName).upload(uploadPath, file, { + contentType: 'image/jpeg', + }) + const err = res.error as StorageApiError + expect(err.toJSON()).toEqual({ + name: 'StorageApiError', + message: 'mime type not supported', + status: 422, + }) + }) + }) + describe('File operations', () => { test('list objects', async () => { await storage.from(bucketName).upload(uploadPath, file) diff --git a/webpack.config.js b/webpack.config.js index 4139d9d..a1832ea 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -3,6 +3,9 @@ const path = require('path') module.exports = { entry: './src/index.ts', + externals: { + 'tus-js-client': 'tus-js-client', + }, output: { path: path.resolve(__dirname, 'dist/umd'), filename: 'supabase.js',