Writing Cloud files (#9686)

- Closes #9291

Author: radeusgd

CHANGELOG.md

@@ -649,6 +649,7 @@
 - [Added `Decimal.abs`, `.negate` and `.signum`.][9641]
 - [Added `Decimal.min` and `.max`.][9663]
 - [Added `Decimal.round`.][9672]
+- [Implemented write support for Enso Cloud files.][9686]
 
 [debug-shortcuts]:
   https://github.com/enso-org/enso/blob/develop/app/gui/docs/product/shortcuts.md#debug
@@ -948,6 +949,7 @@
 [9641]: https://github.com/enso-org/enso/pull/9641
 [9663]: https://github.com/enso-org/enso/pull/9663
 [9672]: https://github.com/enso-org/enso/pull/9672
+[9686]: https://github.com/enso-org/enso/pull/9686
 
 #### Enso Compiler
 

distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Cloud_Caching_Settings.enso

@@ -0,0 +1,33 @@
+import project.Data.Time.Duration.Duration
+import project.Enso_Cloud.Internal.Existing_Enso_Asset.Asset_Cache
+import project.Nothing.Nothing
+
+polyglot java import org.enso.base.enso_cloud.CacheSettings
+
+## PRIVATE
+   UNSTABLE
+   ADVANCED
+   Sets for how long is Enso Cloud file information cached without checking for
+   external updates.
+
+   The default TTL is 60 seconds.
+
+   Side effects from this Enso workflow will invalidate the cache immediately,
+   but any external operations (done from other Enso instances) will not be
+   visible until a cached value expires. Thus if the workflow is expected to
+   co-operate with other workflows, it may be useful to decrease the cache TTL
+   or disable it completely by passing `Nothing`.
+
+   Note that completely disabling the caching will affect performance, as some
+   generic operations may perform multiple cloud requests.
+
+   Changing the TTL invalidates all existing cache entries, because their
+   expiration time was calculated using the old TTL.
+set_file_cache_ttl (duration : Duration | Nothing) =
+    CacheSettings.setFileCacheTTL duration
+    Asset_Cache.invalidate_all
+
+## PRIVATE
+   ADVANCED
+   Returns the current file cache TTL.
+get_file_cache_ttl -> Duration | Nothing = CacheSettings.getFileCacheTTL

distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Enso_File.enso

@@ -10,7 +10,9 @@ import project.Enso_Cloud.Data_Link
 import project.Enso_Cloud.Enso_User.Enso_User
 import project.Enso_Cloud.Errors.Enso_Cloud_Error
 import project.Enso_Cloud.Internal.Enso_Path.Enso_Path
+import project.Enso_Cloud.Internal.Enso_File_Write_Strategy
 import project.Enso_Cloud.Internal.Existing_Enso_Asset.Existing_Enso_Asset
+import project.Enso_Cloud.Internal.Existing_Enso_Asset.Asset_Cache
 import project.Enso_Cloud.Internal.Utils
 import project.Error.Error
 import project.Errors.Common.Not_Found
@@ -21,19 +23,23 @@ import project.Errors.Time_Error.Time_Error
 import project.Errors.Unimplemented.Unimplemented
 import project.Network.HTTP.HTTP
 import project.Network.HTTP.HTTP_Method.HTTP_Method
+import project.Network.HTTP.Request_Error
 import project.Network.URI.URI
 import project.Nothing.Nothing
+import project.Panic.Panic
 import project.Runtime
 import project.Runtime.Context
 import project.System.Environment
 import project.System.File.Data_Link_Access.Data_Link_Access
+import project.System.File.File
 import project.System.File.File_Access.File_Access
 import project.System.File.Generic.File_Like.File_Like
 import project.System.File.Generic.Writable_File.Writable_File
 import project.System.File_Format_Metadata.File_Format_Metadata
 import project.System.Input_Stream.Input_Stream
 import project.System.Output_Stream.Output_Stream
 from project.Data.Boolean import Boolean, False, True
+from project.Data.Index_Sub_Range.Index_Sub_Range import Last
 from project.Data.Text.Extensions import all
 from project.Enso_Cloud.Public_Utils import get_required_field
 from project.System.File_Format import Auto_Detect, Bytes, File_Format, Plain_Text_Format
@@ -178,8 +184,17 @@ type Enso_File
          value. The value is returned from this method.
     with_output_stream : Vector File_Access -> (Output_Stream -> Any ! File_Error) -> Any ! File_Error
     with_output_stream self (open_options : Vector) action =
-        _ = [open_options, action]
-        Unimplemented.throw "Writing to Enso_Files is not currently implemented."
+        Context.Output.if_enabled disabled_message="Writing to an Enso_File is forbidden as the Output context is disabled." panic=False <|
+            open_as_data_link = (open_options.contains Data_Link_Access.No_Follow . not) && (Data_Link.is_data_link self)
+            if open_as_data_link then Data_Link.write_data_link_as_stream self open_options action else
+                if open_options.contains File_Access.Append then Unimplemented.throw "Enso_File currently does not support appending to a file. Instead you may read it, modify and then write the new contents." else
+                    File_Access.ensure_only_allowed_options "with_output_stream" [File_Access.Write, File_Access.Create_New, File_Access.Truncate_Existing, File_Access.Create, Data_Link_Access.No_Follow] open_options <|
+                        allow_existing = open_options.contains File_Access.Create_New . not
+                        tmp_file = File.create_temporary_file "enso-cloud-write-tmp"
+                        Panic.with_finalizer tmp_file.delete <|
+                            perform_upload self allow_existing <|
+                                result = tmp_file.with_output_stream [File_Access.Write] action
+                                result.if_not_error [tmp_file, result]
 
     ## PRIVATE
        ADVANCED
@@ -240,7 +255,6 @@ type Enso_File
             Enso_Asset_Type.File -> File_Format.handle_format_missing_arguments format <|
                 read_with_format effective_format =
                     metadata = File_Format_Metadata.from self
-                    # TODO maybe avoid fetching asset id twice here? maybe move with_input_stream to asset? or helper?
                     self.with_input_stream [File_Access.Read] (stream-> effective_format.read_stream stream metadata)
 
                 if format != Auto_Detect then read_with_format format else
@@ -292,33 +306,40 @@ type Enso_File
         # Remove secrets from the list - they are handled separately in `Enso_Secret.list`.
         assets = list_assets self . filter f-> f.asset_type != Enso_Asset_Type.Secret
         assets.map asset->
-            # TODO we could cache the asset maybe? but for how long should we do that?
-            Enso_File.Value (self.enso_path.resolve asset.name)
+            file = Enso_File.Value (self.enso_path.resolve asset.name)
+            Asset_Cache.update file asset
+            file
 
     ## UNSTABLE
        GROUP Output
        Creates a subdirectory in a specified directory.
     create_directory : Text -> Enso_File
     create_directory self (name : Text) =
+        effective_name = if name.ends_with "/" then name.drop (Last 1) else name
         asset = Existing_Enso_Asset.get_asset_reference_for self
         if asset.is_directory.not then Error.throw (Illegal_Argument.Error "Only directories can contain subdirectories.") else
             parent_field = if self.is_current_user_root then [] else
-                [["parentId", [asset.id]]]
-            body = JS_Object.from_pairs [["title", name]]+parent_field
+                [["parentId", asset.id]]
+            body = JS_Object.from_pairs [["title", effective_name]]+parent_field
+            created_directory = Enso_File.Value (self.enso_path.resolve name)
+            Asset_Cache.invalidate created_directory
             response = Utils.http_request_as_json HTTP_Method.Post Utils.directory_api body
-            # TODO we could cache the asset maybe? but for how long should we do that?
             created_asset = Existing_Enso_Asset.from_json response
             created_asset.if_not_error <|
-                Enso_File.Value (self.enso_path.resolve name)
+                Asset_Cache.update created_directory created_asset
+                created_directory
 
     ## UNSTABLE
        GROUP Output
        Deletes the file or directory.
     delete : Nothing
     delete self = if self.enso_path.is_root then Error.throw (Illegal_Argument.Error "The root directory cannot be deleted.") else
         asset = Existing_Enso_Asset.get_asset_reference_for self
-        uri = URI.from asset.asset_uri . add_query_argument "force" "true"
+        # The `if_not_error` here is a workaround for bug https://github.com/enso-org/enso/issues/9669
+        uri = asset.if_not_error <|
+            URI.from asset.asset_uri . add_query_argument "force" "true"
         response = Utils.http_request HTTP_Method.Delete uri
+        if asset.is_directory then Asset_Cache.invalidate_subtree self else Asset_Cache.invalidate self
         response.if_not_error Nothing
 
     ## ICON data_output
@@ -350,9 +371,9 @@ type Enso_File
          destination file already exists. Defaults to `False`.
     move_to : Writable_File -> Boolean -> Nothing ! File_Error
     move_to self (destination : Writable_File) (replace_existing : Boolean = False) =
-        _ = [destination, replace_existing]
-        Context.Output.if_enabled disabled_message="File moving is forbidden as the Output context is disabled." panic=False <|
-            Unimplemented.throw "Enso_File.move_to is not implemented"
+        # TODO we could have a fast path if Cloud will support renaming files
+        generic_copy self destination.file replace_existing . if_not_error <|
+            self.delete . if_not_error <| destination.file
 
     ## GROUP Operators
        ICON folder
@@ -371,6 +392,11 @@ type Enso_File
     to_text self -> Text =
         "Enso_File "+self.path
 
+    ## PRIVATE
+    to_js_object : JS_Object
+    to_js_object self =
+        JS_Object.from_pairs [["type", "Enso_File"], ["constructor", "new"], ["path", self.path.to_text]]
+
 ## PRIVATE
 list_assets (parent : Enso_File) -> Vector Existing_Enso_Asset =
     Existing_Enso_Asset.get_asset_reference_for parent . list_directory
@@ -402,16 +428,53 @@ Enso_Asset_Type.from (that:Text) = case that of
 
 ## PRIVATE
 File_Format_Metadata.from (that:Enso_File) =
-    # TODO this is just a placeholder, until we implement the proper path
-    path = Nothing
-    case that.asset_type of
-        Enso_Asset_Type.Data_Link ->
-            File_Format_Metadata.Value path=path name=that.name content_type=Data_Link.data_link_content_type
-        Enso_Asset_Type.Directory ->
-            File_Format_Metadata.Value path=path name=that.name extension=(that.extension.catch _->Nothing)
-        Enso_Asset_Type.File ->
-            File_Format_Metadata.Value path=path name=that.name extension=(that.extension.catch _->Nothing)
-        other_asset_type -> Error.throw (Illegal_Argument.Error "`File_Format_Metadata` is not available for: "+other_asset_type.to_text+".")
+    asset_type = that.asset_type.catch File_Error _->Nothing
+    if asset_type == Enso_Asset_Type.Data_Link then File_Format_Metadata.Value path=that.path name=that.name content_type=Data_Link.data_link_content_type else
+        File_Format_Metadata.Value path=that.path name=that.name extension=(that.extension.catch _->Nothing)
 
 ## PRIVATE
 File_Like.from (that : Enso_File) = File_Like.Value that
+
+## PRIVATE
+Writable_File.from (that : Enso_File) =
+    Writable_File.Value that Enso_File_Write_Strategy.instance
+
+## PRIVATE
+upload_file (local_file : File) (destination : Enso_File) (replace_existing : Boolean) -> Enso_File =
+    result = perform_upload destination replace_existing [local_file, destination]
+    result.catch Enso_Cloud_Error error->
+        is_source_file_not_found = case error of
+            Enso_Cloud_Error.Connection_Error cause -> case cause of
+                request_error : Request_Error -> request_error.error_type == 'java.io.FileNotFoundException'
+                _ -> False
+            _ -> False
+        if is_source_file_not_found then Error.throw (File_Error.Not_Found local_file) else result
+
+## PRIVATE
+   `generate_request_body_and_result` should return a pair,
+   where the first element is the request body and the second element is the result to be returned.
+   It is executed lazily, only after all pre-conditions are successfully met.
+perform_upload (destination : Enso_File) (allow_existing : Boolean) (~generate_request_body_and_result) =
+    parent_directory = destination.parent
+    if parent_directory.is_nothing then Error.throw (Illegal_Argument.Error "The root directory cannot be a destination for upload. The destination must be a path to a file.") else
+        parent_directory_asset = Existing_Enso_Asset.get_asset_reference_for parent_directory
+        # If the parent directory does not exist, we fail.
+        parent_directory_asset.if_not_error <|
+            existing_asset = Existing_Enso_Asset.get_asset_reference_for destination
+                . catch File_Error _->Nothing
+            if existing_asset.is_nothing.not && allow_existing.not then Error.throw (File_Error.Already_Exists destination) else
+                if existing_asset.is_nothing.not && existing_asset.asset_type != Enso_Asset_Type.File then Error.throw (Illegal_Argument.Error "The destination must be a path to a file, not "+existing_asset.asset_type.to_text+".") else
+                    existing_asset_id = existing_asset.if_not_nothing <| existing_asset.id
+                    file_name = destination.name
+                    base_uri = URI.from Utils.files_api
+                        . add_query_argument "parent_directory_id" parent_directory_asset.id
+                        . add_query_argument "file_name" file_name
+                    full_uri = case existing_asset_id of
+                        Nothing -> base_uri
+                        _ -> base_uri . add_query_argument "file_id" existing_asset_id
+                    pair = generate_request_body_and_result
+                    Asset_Cache.invalidate destination
+                    response = Utils.http_request HTTP_Method.Post full_uri pair.first
+                    response.if_not_error <|
+                        Asset_Cache.update destination (Existing_Enso_Asset.from_json response)
+                        pair.second

distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Internal/Authentication.enso

@@ -27,6 +27,7 @@ import project.Network.HTTP.Response.Response
 import project.Network.URI.URI
 import project.Nothing.Nothing
 import project.Panic.Panic
+import project.Runtime.Context
 import project.Runtime.Ref.Ref
 import project.System.Environment
 import project.System.File.File
@@ -135,7 +136,7 @@ type Refresh_Token_Data
         headers = [Header.content_type "application/x-amz-json-1.1", Header.new "X-Amz-Target" "AWSCognitoIdentityProviderService.InitiateAuth"]
         auth_parameters = JS_Object.from_pairs [["REFRESH_TOKEN", self.refresh_token], ["DEVICE_KEY", Nothing]]
         payload = JS_Object.from_pairs [["ClientId", self.client_id], ["AuthFlow", "REFRESH_TOKEN_AUTH"], ["AuthParameters", auth_parameters]]
-        response = HTTP.post self.refresh_url body=(Request_Body.Json payload) headers=headers
+        response = Context.Output.with_enabled <| HTTP.post self.refresh_url body=(Request_Body.Json payload) headers=headers
             . catch HTTP_Error error-> case error of
                 HTTP_Error.Status_Error status _ _ ->
                     # If the status code is 400-499, then most likely the reason is that the session has expired, so we ask the user to log in again.
@@ -179,4 +180,6 @@ file_get_required_string_field json prefix field_name = case json of
             _ ->
                 got_type = Meta.type_of result . to_display_text
                 Panic.throw (Illegal_State.Error prefix+"expected `"+field_name+"` to be a string, but got "+got_type+".")
-    _ -> Panic.throw (Illegal_State.Error prefix+"expected an object, got "+(Meta.type_of json))
+    _ ->
+        got_type = Meta.type_of json . to_display_text
+        Panic.throw (Illegal_State.Error prefix+"expected an object, got "+got_type)

distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Internal/Enso_File_Write_Strategy.enso

@@ -0,0 +1,34 @@
+private
+
+import project.Error.Error
+import project.Errors.Common.Forbidden_Operation
+import project.Errors.Unimplemented.Unimplemented
+import project.System.File.Existing_File_Behavior.Existing_File_Behavior
+import project.System.File.File
+from project.Data.Boolean import Boolean, False, True
+from project.Enso_Cloud.Enso_File import Enso_File, upload_file
+from project.System.File.Generic.File_Write_Strategy import default_append, default_overwrite, default_raise_error, File_Write_Strategy, generic_remote_write_with_local_file
+
+## PRIVATE
+   In the Enso_File, we use the Overwrite strategy for Backup.
+   That is because, the Cloud keeps versions of the file by itself,
+   so there is no need to duplicate its work on our own - just overwriting the
+   file still ensures we have a backup.
+instance =
+    File_Write_Strategy.Value default_overwrite default_append default_raise_error write_backing_up=default_overwrite create_dry_run_file remote_write_with_local_file copy_from_local
+
+
+## PRIVATE
+create_dry_run_file file copy_original =
+    _ = [file, copy_original]
+    Error.throw (Forbidden_Operation.Error "Currently dry-run is not supported for Enso_File, so writing to an Enso_File is forbidden if the Output context is disabled.")
+
+
+## PRIVATE
+remote_write_with_local_file file existing_file_behavior action =
+    if existing_file_behavior == Existing_File_Behavior.Append then Unimplemented.throw "Enso Cloud does not yet support appending to a file. Instead you may read it, modify and then write the new contents." else
+        generic_remote_write_with_local_file file existing_file_behavior action
+
+## PRIVATE
+copy_from_local (source : File) (destination : Enso_File) (replace_existing : Boolean) =
+    upload_file source destination replace_existing

distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Internal/Existing_Enso_Asset.enso

@@ -7,6 +7,7 @@ import project.Data.Text.Text_Sub_Range.Text_Sub_Range
 import project.Data.Time.Date_Time.Date_Time
 import project.Data.Time.Date_Time_Formatter.Date_Time_Formatter
 import project.Data.Vector.Vector
+import project.Enso_Cloud.Cloud_Caching_Settings
 import project.Enso_Cloud.Enso_User.Enso_User
 import project.Enso_Cloud.Enso_File.Enso_Asset_Type
 import project.Enso_Cloud.Enso_File.Enso_File
@@ -73,9 +74,7 @@ type Existing_Enso_Asset
        Fetches the basic information about an existing file from the Cloud.
        It will fail if the file does not exist.
     get_asset_reference_for (file : Enso_File) -> Existing_Enso_Asset ! File_Error =
-        # TODO remove workaround for bug https://github.com/enso-org/cloud-v2/issues/1173
-        path = if file.enso_path.is_root then file.enso_path.to_text + "/" else file.enso_path.to_text
-        Existing_Enso_Asset.resolve_path path if_not_found=(Error.throw (File_Error.Not_Found file))
+        fetch_asset_reference file
 
     ## PRIVATE
        Resolves a path to an existing asset in the cloud.
@@ -111,3 +110,31 @@ type Existing_Enso_Asset
         #org = json.get "organizationId" ""
         asset_type = (id.take (Text_Sub_Range.Before "-")):Enso_Asset_Type
         Existing_Enso_Asset.Value title id asset_type
+
+
+## PRIVATE
+type Asset_Cache
+    asset_prefix = "asset:"
+
+    asset_key (file : Enso_File) -> Text = Asset_Cache.asset_prefix+file.enso_path.to_text
+
+    ## PRIVATE
+    invalidate (file : Enso_File) =
+        Utils.invalidate_cache (Asset_Cache.asset_key file)
+
+    invalidate_subtree (file : Enso_File) =
+        Utils.invalidate_caches_with_prefix (Asset_Cache.asset_key file)
+
+    invalidate_all =
+        Utils.invalidate_caches_with_prefix Asset_Cache.asset_prefix
+
+    update (file : Enso_File) (asset : Existing_Enso_Asset) =
+        Utils.set_cached (Asset_Cache.asset_key file) asset cache_duration=Cloud_Caching_Settings.get_file_cache_ttl
+
+## PRIVATE
+   Returns the cached reference or fetches it from the cloud.
+fetch_asset_reference (file : Enso_File) -> Existing_Enso_Asset ! File_Error =
+    # TODO remove workaround for bug https://github.com/enso-org/cloud-v2/issues/1173
+    path = if file.enso_path.is_root then file.enso_path.to_text + "/" else file.enso_path.to_text
+    Utils.get_cached (Asset_Cache.asset_key file) cache_duration=Cloud_Caching_Settings.get_file_cache_ttl <|
+        Existing_Enso_Asset.resolve_path path if_not_found=(Error.throw (File_Error.Not_Found file))