Multipart upload for large files (#11319)

- Close enso-org/cloud-v2#1492
- Use multiplart upload for large files (>6MB)
- Split file into chunks of *exactly* 10MB (backend currently returns a fixed number of chunks)
- Call "start upload", then upload each chunk to S3 (currently sequentially), then "end upload"
- Files <=6MB (lambda limit) still use the current endpoint

# Important Notes
None

app/common/src/services/Backend.ts

@@ -6,6 +6,9 @@ import * as newtype from '../utilities/data/newtype'
 import * as permissions from '../utilities/permissions'
 import * as uniqueString from '../utilities/uniqueString'
 
+/** The size, in bytes, of the chunks which the backend accepts. */
+export const S3_CHUNK_SIZE_BYTES = 10_000_000
+
 // ================
 // === Newtypes ===
 // ================
@@ -1236,6 +1239,50 @@ export interface UploadFileRequestParams {
   readonly parentDirectoryId: DirectoryId | null
 }
 
+/** HTTP request body for the "upload file start" endpoint. */
+export interface UploadFileStartRequestBody {
+  readonly size: number
+  readonly fileName: string
+}
+
+/** Metadata required to uploading a large file. */
+export interface UploadLargeFileMetadata {
+  readonly presignedUrls: readonly HttpsUrl[]
+  readonly uploadId: string
+  readonly sourcePath: S3FilePath
+}
+
+/** Metadata for each multipart upload. */
+export interface S3MultipartPart {
+  readonly eTag: string
+  readonly partNumber: number
+}
+
+/** HTTP request body for the "upload file end" endpoint. */
+export interface UploadFileEndRequestBody {
+  readonly parentDirectoryId: DirectoryId | null
+  readonly parts: readonly S3MultipartPart[]
+  readonly sourcePath: S3FilePath
+  readonly uploadId: string
+  readonly assetId: AssetId | null
+  readonly fileName: string
+}
+
+/** A large file that has finished uploading. */
+export interface UploadedLargeFile {
+  readonly id: FileId
+  readonly project: null
+}
+
+/** A large project that has finished uploading. */
+export interface UploadedLargeProject {
+  readonly id: ProjectId
+  readonly project: Project
+}
+
+/** A large asset (file or project) that has finished uploading. */
+export type UploadedLargeAsset = UploadedLargeFile | UploadedLargeProject
+
 /** URL query string parameters for the "upload profile picture" endpoint. */
 export interface UploadPictureRequestParams {
   readonly fileName: string | null
@@ -1520,8 +1567,15 @@ export default abstract class Backend {
   abstract checkResources(projectId: ProjectId, title: string): Promise<ResourceUsage>
   /** Return a list of files accessible by the current user. */
   abstract listFiles(): Promise<readonly FileLocator[]>
-  /** Upload a file. */
-  abstract uploadFile(params: UploadFileRequestParams, file: Blob): Promise<FileInfo>
+  /** Begin uploading a large file. */
+  abstract uploadFileStart(
+    body: UploadFileRequestParams,
+    file: File,
+  ): Promise<UploadLargeFileMetadata>
+  /** Upload a chunk of a large file. */
+  abstract uploadFileChunk(url: HttpsUrl, file: Blob, index: number): Promise<S3MultipartPart>
+  /** Finish uploading a large file. */
+  abstract uploadFileEnd(body: UploadFileEndRequestBody): Promise<UploadedLargeAsset>
   /** Change the name of a file. */
   abstract updateFile(fileId: FileId, body: UpdateFileRequestBody, title: string): Promise<void>
   /** Return file details. */

app/common/src/text/english.json

@@ -142,6 +142,9 @@
   "checkResourcesBackendError": "Could not get resource usage for project '$0'",
   "listFilesBackendError": "Could not list files",
   "uploadFileBackendError": "Could not upload file",
+  "uploadFileStartBackendError": "Could not begin uploading large file",
+  "uploadFileChunkBackendError": "Could not upload chunk of large file",
+  "uploadFileEndBackendError": "Could not finish uploading large file",
   "updateFileNotImplementedBackendError": "Files currently cannot be renamed on the Cloud backend",
   "uploadFileWithNameBackendError": "Could not upload file '$0'",
   "getFileDetailsBackendError": "Could not get details of project '$0'",
@@ -271,6 +274,7 @@
   "options": "Options",
   "googleIcon": "Google icon",
   "gitHubIcon": "GitHub icon",
+  "close": "Close",
 
   "enterSecretPath": "Enter secret path",
   "enterText": "Enter text",
@@ -481,6 +485,11 @@
   "youHaveNoUserGroupsAdmin": "This organization has no user groups. You can create one using the button above.",
   "youHaveNoUserGroupsNonAdmin": "This organization has no user groups. You can create one using the button above.",
   "xIsUsingTheProject": "'$0' is currently using the project",
+  "uploadLargeFileStatus": "Uploading file... ($0/$1MB)",
+  "uploadLargeFileSuccess": "Finished uploading file.",
+  "uploadLargeFileError": "Could not upload file",
+  "closeWindowDialogTitle": "Close window?",
+  "anUploadIsInProgress": "An upload is in progress.",
 
   "enableMultitabs": "Enable Multi-Tabs",
   "enableMultitabsDescription": "Open multiple projects at the same time.",

app/common/src/text/index.ts

@@ -144,6 +144,7 @@ interface PlaceholderOverrides {
 
   readonly arbitraryFieldTooLarge: [maxSize: string]
   readonly arbitraryFieldTooSmall: [minSize: string]
+  readonly uploadLargeFileStatus: [uploadedParts: number, totalParts: number]
 }
 
 /** An tuple of `string` for placeholders for each {@link TextId}. */

app/gui/e2e/dashboard/api.ts

@@ -38,6 +38,7 @@ const GLOB_TAG_ID = backend.TagId('*')
 const GLOB_CHECKOUT_SESSION_ID = backend.CheckoutSessionId('*')
 /* eslint-enable no-restricted-syntax */
 const BASE_URL = 'https://mock/'
+const MOCK_S3_BUCKET_URL = 'https://mock-s3-bucket.com/'
 
 // ===============
 // === mockApi ===
@@ -723,30 +724,39 @@ async function mockApiInternal({ page, setupAPI }: MockParams) {
         return
       }
     })
-    await post(remoteBackendPaths.UPLOAD_FILE_PATH + '*', (_route, request) => {
-      /** The type for the JSON request payload for this endpoint. */
-      interface SearchParams {
-        // eslint-disable-next-line @typescript-eslint/naming-convention
-        readonly file_name: string
-        // eslint-disable-next-line @typescript-eslint/naming-convention
-        readonly file_id?: backend.FileId
-        // eslint-disable-next-line @typescript-eslint/naming-convention
-        readonly parent_directory_id?: backend.DirectoryId
+    await page.route(MOCK_S3_BUCKET_URL + '**', async (route, request) => {
+      if (request.method() !== 'PUT') {
+        await route.fallback()
+      } else {
+        await route.fulfill({
+          headers: {
+            'Access-Control-Expose-Headers': 'ETag',
+            ETag: uniqueString.uniqueString(),
+          },
+        })
       }
+    })
+    await post(remoteBackendPaths.UPLOAD_FILE_START_PATH + '*', () => {
+      return {
+        sourcePath: backend.S3FilePath(''),
+        uploadId: 'file-' + uniqueString.uniqueString(),
+        presignedUrls: Array.from({ length: 10 }, () =>
+          backend.HttpsUrl(`${MOCK_S3_BUCKET_URL}${uniqueString.uniqueString()}`),
+        ),
+      } satisfies backend.UploadLargeFileMetadata
+    })
+    await post(remoteBackendPaths.UPLOAD_FILE_END_PATH + '*', (_route, request) => {
       // The type of the search params sent by this app is statically known.
-      // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, no-restricted-syntax
-      const searchParams: SearchParams = Object.fromEntries(
-        new URL(request.url()).searchParams.entries(),
-      ) as never
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+      const body: backend.UploadFileEndRequestBody = request.postDataJSON()
 
-      const file = addFile(
-        searchParams.file_name,
-        searchParams.parent_directory_id == null ?
-          {}
-        : { parentId: searchParams.parent_directory_id },
-      )
+      const file = addFile(body.fileName, {
+        id: backend.FileId(body.uploadId),
+        title: body.fileName,
+        ...(body.parentDirectoryId != null ? { parentId: body.parentDirectoryId } : {}),
+      })
 
-      return { path: '', id: file.id, project: null } satisfies backend.FileInfo
+      return { id: file.id, project: null } satisfies backend.UploadedLargeAsset
     })
 
     await post(remoteBackendPaths.CREATE_SECRET_PATH + '*', async (_route, request) => {

app/gui/src/dashboard/components/JSONSchemaInput.tsx

@@ -6,7 +6,7 @@ import { Button, Checkbox, Dropdown, Text } from '#/components/AriaComponents'
 import Autocomplete from '#/components/Autocomplete'
 import FocusRing from '#/components/styled/FocusRing'
 import { useBackendQuery } from '#/hooks/backendHooks'
-import { useRemoteBackendStrict } from '#/providers/BackendProvider'
+import { useRemoteBackend } from '#/providers/BackendProvider'
 import { useText } from '#/providers/TextProvider'
 import { constantValueOfSchema, getSchemaName, lookupDef } from '#/utilities/jsonSchema'
 import { asObject, singletonObjectOrNull } from '#/utilities/object'
@@ -37,7 +37,7 @@ export default function JSONSchemaInput(props: JSONSchemaInputProps) {
   const { noBorder = false, isAbsent = false, value, onChange } = props
   // The functionality for inputting `enso-secret`s SHOULD be injected using a plugin,
   // but it is more convenient to avoid having plugin infrastructure.
-  const remoteBackend = useRemoteBackendStrict()
+  const remoteBackend = useRemoteBackend()
   const { getText } = useText()
   const [autocompleteText, setAutocompleteText] = useState(() =>
     typeof value === 'string' ? value : null,