Run cloud project locally (#12198)

* feat: RemoteBackend upload

* DRAFT: run locally

* feat: download project middleware

* fix: project downloading

* feat: recover missing project metadata

* fix: open project

* update: move project opening logic to menu

* misc: format

* fix: lint

* fix: open project

* feat: add feature flag

* feat: upload project

* misc: cleanup

* update: simplify download request

* update: simplify upload

* update: download directory name

* update: use crypto

* update: failure handling todo

* update: list projects with ensureQueryData

* update: downloadProject error

* misc: format

* update: project schema

* misc: fixes

* fix: lint

* fix: after merge

Author: 4e6

app/common/src/text/english.json

@@ -810,7 +810,7 @@
   "settingsShortcut": "Settings",
   "closeTabShortcut": "Close Tab",
   "openShortcut": "Open",
-  "runShortcut": "Execute as Task",
+  "runShortcut": "Run Locally",
   "closeShortcut": "Close",
   "uploadToCloudShortcut": "Upload to Cloud",
   "renameShortcut": "Rename",

app/gui/project-manager-shim-middleware/index.ts

@@ -2,9 +2,11 @@
  * @file A HTTP server middleware which handles routes normally proxied through to
  * the Project Manager.
  */
+import * as crypto from 'node:crypto'
 import * as fsSync from 'node:fs'
 import * as fs from 'node:fs/promises'
 import * as http from 'node:http'
+import * as https from 'node:https'
 import * as path from 'node:path'
 
 import * as tar from 'tar'
@@ -21,6 +23,7 @@ import * as projectManagement from './projectManagement'
 const HTTP_STATUS_OK = 200
 const HTTP_STATUS_BAD_REQUEST = 400
 const HTTP_STATUS_NOT_FOUND = 404
+const HTTP_STATUS_INTERNAL_SERVER_ERROR = 500
 const PROJECTS_ROOT_DIRECTORY = projectManagement.getProjectsDirectory()
 
 const COMMON_HEADERS = {
@@ -135,6 +138,106 @@ export default function projectManagerShimMiddleware(
       ),
       { end: true },
     )
+  } else if (requestUrl != null && requestUrl.startsWith('/api/cloud/')) {
+    switch (requestPath) {
+      case '/api/cloud/download-project': {
+        const url = new URL(`https://example.com/${requestUrl}`)
+        const downloadUrl = url.searchParams.get('downloadUrl')
+        const projectId = url.searchParams.get('projectId')
+
+        if (downloadUrl == null) {
+          response
+            .writeHead(HTTP_STATUS_BAD_REQUEST, COMMON_HEADERS)
+            .end('Request is missing search parameter `downloadUrl`.')
+          break
+        }
+
+        if (projectId == null) {
+          response
+            .writeHead(HTTP_STATUS_BAD_REQUEST, COMMON_HEADERS)
+            .end('Request is missing search parameter `projectId`.')
+          break
+        }
+
+        https.get(downloadUrl, (actualResponse) => {
+          const projectsDirectory = projectManagement.getProjectsDirectory()
+          const targetDirectory = path.join(projectsDirectory, `cloud-${projectId}`)
+
+          fs.mkdir(targetDirectory, { recursive: true })
+            .then(() => projectManagement.unpackBundle(actualResponse, targetDirectory))
+            .then((projectDirectory) => {
+              response.writeHead(HTTP_STATUS_OK, COMMON_HEADERS).end(projectDirectory)
+            })
+            .catch((e) => {
+              console.error(e)
+              response.writeHead(HTTP_STATUS_INTERNAL_SERVER_ERROR, COMMON_HEADERS).end()
+            })
+        })
+
+        break
+      }
+      case '/api/cloud/upload-project': {
+        const url = new URL(`https://example.com/${requestUrl}`)
+        const uploadUrl = url.searchParams.get('uploadUrl')
+        const projectDir = url.searchParams.get('directory')
+
+        if (uploadUrl == null) {
+          response
+            .writeHead(HTTP_STATUS_BAD_REQUEST, COMMON_HEADERS)
+            .end('Request is missing search parameter `uploadUrl`.')
+          break
+        }
+        if (projectDir == null) {
+          response
+            .writeHead(HTTP_STATUS_BAD_REQUEST, COMMON_HEADERS)
+            .end('Request is missing search parameter `directory`.')
+          break
+        }
+
+        projectManagement
+          .createBundle(projectDir)
+          .then((projectBundle) => {
+            const headers = {
+              authorization: request.headers.authorization,
+            }
+            const uploadRequest = https.request(
+              uploadUrl,
+              { method: 'POST', headers },
+              (actualResponse) => {
+                if (!response.writableFinished) {
+                  response.writeHead(
+                    // This is SAFE. The documentation says:
+                    // Only valid for response obtained from ClientRequest.
+                    actualResponse.statusCode!,
+                    actualResponse.statusMessage,
+                    actualResponse.headers,
+                  )
+                  actualResponse.pipe(response, { end: true })
+                }
+              },
+            )
+            uploadRequest.write(projectBundle, (err) => {
+              if (err) {
+                console.error(err)
+                response
+                  .writeHead(HTTP_STATUS_INTERNAL_SERVER_ERROR)
+                  .end('Failed to write project bundle.')
+              }
+            })
+            uploadRequest.end()
+          })
+          .catch((err) => {
+            console.error(err)
+            response.writeHead(HTTP_STATUS_INTERNAL_SERVER_ERROR, COMMON_HEADERS).end()
+          })
+
+        break
+      }
+      default: {
+        console.error(`Unknown Cloud middleware request:`, requestPath)
+        break
+      }
+    }
   } else if (request.method === 'POST') {
     switch (requestPath) {
       case '/api/upload-file': {
@@ -251,10 +354,30 @@ export default function projectManagerShimMiddleware(
                             projectManagement.PROJECT_METADATA_RELATIVE_PATH,
                           )
                           const packageMetadataContents = await fs.readFile(packageMetadataPath)
-                          const projectMetadataContents = await fs.readFile(projectMetadataPath)
+                          const packageMetadataYaml = yaml.parse(packageMetadataContents.toString())
+                          let projectMetadataJson
+                          try {
+                            const projectMetadataContents = await fs.readFile(projectMetadataPath)
+                            projectMetadataJson = JSON.parse(projectMetadataContents.toString())
+                          } catch (e) {
+                            if (
+                              'name' in packageMetadataYaml &&
+                              typeof packageMetadataYaml.name === 'string'
+                            ) {
+                              projectMetadataJson = {
+                                id: crypto.randomUUID(),
+                                kind: 'UserProject',
+                                created: new Date().toISOString(),
+                                lastOpened: null,
+                              }
+                              fs.writeFile(projectMetadataPath, JSON.stringify(projectMetadataJson))
+                            } else {
+                              throw e
+                            }
+                          }
                           const metadata = extractProjectMetadata(
-                            yaml.parse(packageMetadataContents.toString()),
-                            JSON.parse(projectMetadataContents.toString()),
+                            packageMetadataYaml,
+                            projectMetadataJson,
                           )
                           if (metadata != null) {
                             // This is a project.

app/gui/project-manager-shim-middleware/projectManagement.ts

@@ -60,6 +60,40 @@ export async function uploadBundle(
   return bumpMetadata(targetPath, directory, name ?? null)
 }
 
+/** Create a .tar.gz enso-project bundle. */
+export function createBundle(directory: string): Promise<Buffer> {
+  const readableStream = tar.c(
+    {
+      z: true,
+      C: directory,
+    },
+    ['.'],
+  )
+  return new Promise((resolve, reject) => {
+    const chunks: Buffer[] = []
+    readableStream.on('data', (data) => chunks.push(data))
+    readableStream.on('end', () => resolve(Buffer.concat(chunks)))
+    readableStream.on('error', reject)
+  })
+}
+
+/** Unpack a .tar.gz enso-project bundle into a temporary directory */
+export async function unpackBundle(
+  bundle: stream.Readable,
+  targetDirectory: string,
+): Promise<string> {
+  return new Promise((resolve, reject) => {
+    bundle
+      .pipe(
+        tar.x({
+          C: targetDirectory,
+        }),
+      )
+      .on('end', () => resolve(targetDirectory))
+      .on('error', (err) => reject(err))
+  })
+}
+
 // ================
 // === Metadata ===
 // ================

app/gui/src/dashboard/components/Devtools/EnsoDevtoolsImpl.tsx

@@ -280,6 +280,15 @@ export function EnsoDevtools() {
                       setFeatureFlag('enableAdvancedProjectExecutionOptions', value)
                     }}
                   />
+                  <ariaComponents.Switch
+                    form={form}
+                    name="enableHybridExecution"
+                    label="Enable Hybrid Execution"
+                    description="Enable Hybrid Execution"
+                    onChange={(value) => {
+                      setFeatureFlag('enableHybridExecution', value)
+                    }}
+                  />
                 </>
               )}
             </ariaComponents.Form>

app/gui/src/dashboard/hooks/projectHooks.ts

@@ -276,12 +276,22 @@ export function useCloseProjectMutation() {
 
       void client.cancelQueries({ queryKey })
     },
-    onSuccess: async (_, { type, id, parentId }) => {
+    onSuccess: async (_, { type, id, parentId, cloudProjectId }) => {
       await client.resetQueries({ queryKey: createGetProjectDetailsQuery.getQueryKey(id) })
       setProjectAsset(type, id, parentId, (asset) => ({
         ...asset,
         projectState: { ...asset.projectState, type: backendModule.ProjectState.closed },
       }))
+
+      // If the project runs in hybrid execution mode
+      // TODO: implement proper handling of upload failures
+      if (cloudProjectId) {
+        invariant(localBackend != null, 'LocalBackend is null')
+
+        await remoteBackend.uploadProject(cloudProjectId, parentId)
+
+        await localBackend.deleteAsset(parentId, { force: true }, null)
+      }
     },
     onError: async (_, { type, id, parentId }) => {
       await client.invalidateQueries({ queryKey: createGetProjectDetailsQuery.getQueryKey(id) })

app/gui/src/dashboard/layouts/AssetContextMenu.tsx

@@ -1,5 +1,6 @@
 /** @file The context menu for an arbitrary {@link backendModule.Asset}. */
 import * as React from 'react'
+import invariant from 'tiny-invariant'
 
 import * as reactQuery from '@tanstack/react-query'
 import * as toast from 'react-toastify'
@@ -38,6 +39,7 @@ import { useBackendQuery, useNewProject } from '#/hooks/backendHooks'
 import { useUploadFileWithToastMutation } from '#/hooks/backendUploadFilesHooks'
 import { useGetAsset } from '#/layouts/Drive/assetsTableItemsHooks'
 import { usePasteData } from '#/providers/DriveProvider'
+import * as featureFlagsProvider from '#/providers/FeatureFlagsProvider'
 import { computeFullRemotePath } from '#/services/RemoteBackend'
 import { TEAMS_DIRECTORY_ID, USERS_DIRECTORY_ID } from '#/services/remoteBackendPaths'
 import { normalizePath } from '#/utilities/fileInfo'
@@ -87,7 +89,6 @@ export default function AssetContextMenu(props: AssetContextMenuProps) {
   const restoreAssetsMutation = reactQuery.useMutation(restoreAssetsMutationOptions(backend))
   const copyAssetsMutation = reactQuery.useMutation(copyAssetsMutationOptions(backend))
   const downloadAssetsMutation = reactQuery.useMutation(downloadAssetsMutationOptions(backend))
-  const openProjectMutation = projectHooks.useOpenProjectMutation()
   const self = permissions.tryFindSelfPermission(user, asset.permissions)
   const isCloud = categoryModule.isCloudCategory(category)
   const pathComputed =
@@ -170,6 +171,8 @@ export default function AssetContextMenu(props: AssetContextMenuProps) {
     asset.projectState.openedBy != null &&
     asset.projectState.openedBy !== user.email
 
+  const enableHybridExecution = featureFlagsProvider.useFeatureFlag('enableHybridExecution')
+
   const pasteMenuEntry = hasPasteData && canPaste && (
     <ContextMenuEntry
       hidden={hidden}
@@ -245,19 +248,31 @@ export default function AssetContextMenu(props: AssetContextMenuProps) {
               }}
             />
           )}
-        {asset.type === backendModule.AssetType.project && isCloud && (
+        {asset.type === backendModule.AssetType.project && isCloud && enableHybridExecution && (
           <ContextMenuEntry
-            hidden={hidden}
+            hidden={hidden || localBackend == null}
             action="run"
             isDisabled={!canOpenProjects}
             tooltip={disabledTooltip}
-            doAction={() => {
-              openProjectMutation.mutate({
-                id: asset.id,
-                title: asset.title,
-                parentId: asset.parentId,
-                type: state.backend.type,
-                inBackground: true,
+            doAction={async () => {
+              invariant(localBackend != null, 'Local Backend is null')
+              const parentId = await remoteBackend.downloadProject(asset.id)
+              const assets = await localBackend.listDirectory({
+                parentId: parentId,
+                filterBy: null,
+                labels: null,
+                recentProjects: false,
+              })
+              const project = assets
+                .filter((item) => item.type === backendModule.AssetType.project)
+                .at(0)
+              invariant(project, 'Downloaded cloud project does not exist.')
+              openProject({
+                id: project.id,
+                title: project.title,
+                parentId: project.parentId,
+                type: backendModule.BackendType.local,
+                cloudProjectId: asset.id,
               })
             }}
           />

app/gui/src/dashboard/providers/FeatureFlagsProvider.tsx

@@ -31,6 +31,7 @@ export const FEATURE_FLAGS_SCHEMA = z.object({
   enableCloudExecution: z.boolean(),
   enableAsyncExecution: z.boolean(),
   enableAdvancedProjectExecutionOptions: z.boolean(),
+  enableHybridExecution: z.boolean(),
 })
 
 /** Feature flags. */
@@ -56,6 +57,7 @@ const flagsStore = createStore<FeatureFlagsStore>()(
         enableCloudExecution: IS_DEV_MODE || isOnElectron(),
         enableAsyncExecution: true,
         enableAdvancedProjectExecutionOptions: false,
+        enableHybridExecution: IS_DEV_MODE,
       },
       setFeatureFlag: (key, value) => {
         set(({ featureFlags }) => ({ featureFlags: { ...featureFlags, [key]: value } }))

app/gui/src/dashboard/providers/ProjectsProvider.tsx

@@ -33,6 +33,9 @@ const PROJECT_SCHEMA = z
     ),
     title: z.string(),
     type: z.nativeEnum(backendModule.BackendType),
+    cloudProjectId: z.optional(
+      z.custom<backendModule.ProjectId>((x) => typeof x === 'string' && x.startsWith('project-')),
+    ),
   })
   .readonly()
 const LAUNCHED_PROJECT_SCHEMA = z.array(PROJECT_SCHEMA).readonly()

app/gui/src/dashboard/services/RemoteBackend.ts

@@ -1527,6 +1527,38 @@ export default class RemoteBackend extends Backend {
     }
   }
 
+  /** Download the project to a temporary location. */
+  async downloadProject(id: backend.ProjectId): Promise<DirectoryId> {
+    const details = await this.getProjectDetails(id, true)
+
+    invariant(details.url != null, 'The download URL of the project must be present.')
+
+    const queryString = new URLSearchParams({
+      downloadUrl: details.url,
+      projectId: id,
+    })
+
+    const response = await this.client.get(`./api/cloud/download-project?${queryString}`)
+    const path = await response.text()
+
+    if (!response.ok) {
+      return await this.throw(response, 'resolveProjectAssetPathBackendError')
+    }
+
+    return DirectoryId(`directory-${path}` as const)
+  }
+
+  /** Upload the project. */
+  async uploadProject(id: backend.ProjectId, directoryId: backend.DirectoryId): Promise<void> {
+    const uploadPath = remoteBackendPaths.getProjectUploadPath(id)
+    const queryString = new URLSearchParams({
+      uploadUrl: `${$config.API_URL}/${uploadPath}`,
+      directory: extractIdFromDirectoryId(directoryId),
+    })
+
+    await this.client.get(`./api/cloud/upload-project?${queryString}`)
+  }
+
   /** Fetch the URL of the customer portal. */
   override async createCustomerPortalSession() {
     const response = await this.post<backend.CreateCustomerPortalSessionResponse>(

app/gui/src/dashboard/services/remoteBackendPaths.ts

@@ -109,6 +109,10 @@ export function getProjectContentPath(
 export function getProjectAssetPath(projectId: backend.ProjectId, relativePath: string) {
   return `projects/${projectId}/files/${relativePath.replace('./', '')}`
 }
+/** Relative HTTP path to the upload project endpoint of the Cloud backend API. */
+export function getProjectUploadPath(projectId: backend.ProjectId) {
+  return `projects/${projectId}/upload`
+}
 
 /** Relative HTTP path to the "update asset" endpoint of the Cloud backend API. */
 export function updateAssetPath(assetId: backend.AssetId) {