fix: tus-metadata

Author: fenos

src/http/routes/tus/index.ts

@@ -1,9 +1,11 @@
 import { FastifyBaseLogger, FastifyInstance } from 'fastify'
-import { Server } from '@tus/server'
+import * as http from 'http'
+import { Server, ServerOptions, DataStore } from '@tus/server'
 import { jwt, storage, db, dbSuperUser } from '../../plugins'
 import { getConfig } from '../../../config'
-import * as http from 'http'
 import { Storage } from '../../../storage'
+import { FileStore, LockNotifier, PgLocker, S3Store, UploadId } from '../../../storage/tus'
+import { getFileSizeLimit } from '../../../storage/limits'
 import {
   namingFunction,
   onCreate,
@@ -13,15 +15,10 @@ import {
   generateUrl,
   getFileIdFromRequest,
 } from './lifecycle'
-import { ServerOptions, DataStore } from '@tus/server'
-import { getFileSizeLimit } from '../../../storage/limits'
-import { UploadId } from './upload-id'
-import { FileStore } from './file-store'
 import { TenantConnection } from '../../../database/connection'
-import { PgLocker, LockNotifier } from './postgres-locker'
 import { PubSub } from '../../../database/pubsub'
-import { S3Store } from './s3-store'
 import { DeleteHandler } from './handlers'
+import { RequestCacheMetadata } from '../../../storage/tus/request-cache-metadata'
 
 const {
   storageS3Bucket,
@@ -49,6 +46,7 @@ function createTusStore() {
     return new S3Store({
       partSize: 6 * 1024 * 1024, // Each uploaded part will have ~6MB,
       expirationPeriodInMilliseconds: tusUrlExpiryMs,
+      cache: new RequestCacheMetadata(),
       s3ClientConfig: {
         bucket: storageS3Bucket,
         region: storageS3Region,
@@ -126,6 +124,12 @@ export default async function routes(fastify: FastifyInstance) {
     fastify.register(db)
     fastify.register(storage)
 
+    fastify.addHook('onRequest', (req, res, done) => {
+      RequestCacheMetadata.localStorage.run(new Map(), () => {
+        done()
+      })
+    })
+
     fastify.addHook('preHandler', async (req) => {
       ;(req.raw as MultiPartRequest).log = req.log
       ;(req.raw as MultiPartRequest).upload = {

src/http/routes/tus/lifecycle.ts

@@ -1,12 +1,12 @@
 import http from 'http'
-import { isRenderableError, Storage } from '../../../storage'
+import { BaseLogger } from 'pino'
 import { Metadata, Upload } from '@tus/server'
-import { getConfig } from '../../../config'
 import { randomUUID } from 'crypto'
-import { UploadId } from './upload-id'
+import { isRenderableError, Storage, StorageBackendError } from '../../../storage'
+import { getConfig } from '../../../config'
 import { Uploader } from '../../../storage/uploader'
 import { TenantConnection } from '../../../database/connection'
-import { BaseLogger } from 'pino'
+import { UploadId } from '../../../storage/tus'
 
 const { storageS3Bucket, tusPath } = getConfig()
 const reExtractFileID = /([^/]+)\/?$/
@@ -22,6 +22,9 @@ export type MultiPartRequest = http.IncomingMessage & {
   }
 }
 
+/**
+ * Runs on every TUS incoming request
+ */
 export async function onIncomingRequest(
   rawReq: http.IncomingMessage,
   res: http.ServerResponse,
@@ -51,6 +54,9 @@ export async function onIncomingRequest(
   })
 }
 
+/**
+ * Generate URL for TUS upload, it encodes the uploadID to base64url
+ */
 export function generateUrl(
   _: http.IncomingMessage,
   {
@@ -67,6 +73,9 @@ export function generateUrl(
   return `${proto}://${host}${path}/${id}`
 }
 
+/**
+ * Extract the uploadId from the request and decodes it from base64url
+ */
 export function getFileIdFromRequest(rawRwq: http.IncomingMessage) {
   const req = rawRwq as MultiPartRequest
   const match = reExtractFileID.exec(req.url as string)
@@ -79,6 +88,9 @@ export function getFileIdFromRequest(rawRwq: http.IncomingMessage) {
   return req.upload.tenantId + '/' + idMatch
 }
 
+/**
+ * Generate the uploadId for the TUS upload
+ */
 export function namingFunction(rawReq: http.IncomingMessage) {
   const req = rawReq as MultiPartRequest
 
@@ -89,7 +101,7 @@ export function namingFunction(rawReq: http.IncomingMessage) {
   const metadataHeader = req.headers['upload-metadata']
 
   if (typeof metadataHeader !== 'string') {
-    throw new Error('no metadata')
+    throw new StorageBackendError('metadata_header_invalid', 400, 'metadata header invalid')
   }
 
   try {
@@ -108,6 +120,9 @@ export function namingFunction(rawReq: http.IncomingMessage) {
   }
 }
 
+/**
+ * Runs before the upload URL is created
+ */
 export async function onCreate(
   rawReq: http.IncomingMessage,
   res: http.ServerResponse,
@@ -137,6 +152,9 @@ export async function onCreate(
   return res
 }
 
+/**
+ * Runs when the upload to the underline store is completed
+ */
 export async function onUploadFinish(
   rawReq: http.IncomingMessage,
   res: http.ServerResponse,
@@ -178,11 +196,15 @@ export async function onUploadFinish(
 
 type TusError = { status_code: number; body: string }
 
+/**
+ * Runs when there is an error on the TUS upload
+ */
 export function onResponseError(
   req: http.IncomingMessage,
   res: http.ServerResponse,
   e: TusError | Error
 ) {
+  console.log(e)
   if (e instanceof Error) {
     ;(res as any).executionError = e
   }

src/http/routes/tus/s3-store.ts

@@ -2,8 +2,8 @@ import { FileStore as TusFileStore } from '@tus/file-store'
 import { Upload } from '@tus/server'
 import fsExtra from 'fs-extra'
 import path from 'path'
-import { FileBackend } from '../../../storage/backend'
 import { Configstore } from '@tus/file-store'
+import { FileBackend } from '../backend'
 
 type FileStoreOptions = {
   directory: string

src/http/routes/tus/file-store.ts src/storage/tus/file-store.tssrc/http/routes/tus/file-store.ts renamed to src/storage/tus/file-store.ts

@@ -0,0 +1,4 @@
+export * from './file-store'
+export * from './s3-store'
+export * from './postgres-locker'
+export * from './upload-id'

src/storage/tus/index.ts

@@ -1,9 +1,9 @@
 import { Lock, Locker, RequestRelease } from '@tus/server'
-import { Database, DBError } from '../../../storage/database'
-import { PubSubAdapter } from '../../../pubsub'
-import { UploadId } from './upload-id'
 import { clearTimeout } from 'timers'
 import EventEmitter from 'events'
+import { Database, DBError } from '../database'
+import { PubSubAdapter } from '../../pubsub'
+import { UploadId } from './upload-id'
 
 const REQUEST_LOCK_RELEASE_MESSAGE = 'REQUEST_LOCK_RELEASE'
 

src/http/routes/tus/postgres-locker.ts src/storage/tus/postgres-locker.tssrc/http/routes/tus/postgres-locker.ts renamed to src/storage/tus/postgres-locker.ts

@@ -0,0 +1,25 @@
+import { AsyncLocalStorage } from 'async_hooks'
+import { MetadataCache } from './s3-store'
+import { Upload } from '@tus/server'
+
+type MetadataValue = {
+  file: Upload
+  'upload-id': string
+  'tus-version': string
+}
+
+export class RequestCacheMetadata implements MetadataCache {
+  static localStorage = new AsyncLocalStorage<Map<string, MetadataValue>>()
+
+  delete(value: string): Promise<void> | void {
+    RequestCacheMetadata.localStorage.getStore()?.delete(value)
+  }
+
+  get(value: string): Promise<MetadataValue | void> | MetadataValue | void {
+    return RequestCacheMetadata.localStorage.getStore()?.get(value)
+  }
+
+  set(key: string, value: MetadataValue): Promise<void> | void {
+    RequestCacheMetadata.localStorage.getStore()?.set(key, value)
+  }
+}