From fda41cbfd1d1966d8772d1a889905b78651e734e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Tue, 16 Apr 2024 16:01:03 +0200 Subject: [PATCH] Writing Cloud files (#9686) - Closes #9291 --- CHANGELOG.md | 2 + .../Enso_Cloud/Cloud_Caching_Settings.enso | 33 ++ .../0.0.0-dev/src/Enso_Cloud/Enso_File.enso | 109 +++- .../Enso_Cloud/Internal/Authentication.enso | 7 +- .../Internal/Enso_File_Write_Strategy.enso | 34 ++ .../Internal/Existing_Enso_Asset.enso | 33 +- .../src/Enso_Cloud/Internal/Utils.enso | 21 +- .../0.0.0-dev/src/System/File_Format.enso | 5 +- .../enso/base/enso_cloud/CacheSettings.java | 15 + .../base/enso_cloud/CloudRequestCache.java | 24 +- .../Inter_Backend_File_Operations_Spec.enso | 133 ++++- test/AWS_Tests/src/S3_Spec.enso | 2 +- .../Network/Enso_Cloud/Cloud_Tests_Setup.enso | 37 +- .../Network/Enso_Cloud/Enso_Cloud_Spec.enso | 5 +- .../Network/Enso_Cloud/Enso_File_Spec.enso | 508 +++++++++++------- test/Table_Tests/src/IO/Cloud_Spec.enso | 28 + test/Table_Tests/src/IO/Main.enso | 2 + 17 files changed, 722 insertions(+), 276 deletions(-) create mode 100644 distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Cloud_Caching_Settings.enso create mode 100644 distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Internal/Enso_File_Write_Strategy.enso create mode 100644 std-bits/base/src/main/java/org/enso/base/enso_cloud/CacheSettings.java create mode 100644 test/Table_Tests/src/IO/Cloud_Spec.enso diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c35ed90ccc4..fbe14217a30b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -649,6 +649,7 @@ - [Added `Decimal.abs`, `.negate` and `.signum`.][9641] - [Added `Decimal.min` and `.max`.][9663] - [Added `Decimal.round`.][9672] +- [Implemented write support for Enso Cloud files.][9686] [debug-shortcuts]: https://github.com/enso-org/enso/blob/develop/app/gui/docs/product/shortcuts.md#debug @@ -948,6 +949,7 @@ [9641]: https://github.com/enso-org/enso/pull/9641 [9663]: https://github.com/enso-org/enso/pull/9663 [9672]: https://github.com/enso-org/enso/pull/9672 +[9686]: https://github.com/enso-org/enso/pull/9686 #### Enso Compiler diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Cloud_Caching_Settings.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Cloud_Caching_Settings.enso new file mode 100644 index 000000000000..c4dd0e3f9517 --- /dev/null +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Cloud_Caching_Settings.enso @@ -0,0 +1,33 @@ +import project.Data.Time.Duration.Duration +import project.Enso_Cloud.Internal.Existing_Enso_Asset.Asset_Cache +import project.Nothing.Nothing + +polyglot java import org.enso.base.enso_cloud.CacheSettings + +## PRIVATE + UNSTABLE + ADVANCED + Sets for how long is Enso Cloud file information cached without checking for + external updates. + + The default TTL is 60 seconds. + + Side effects from this Enso workflow will invalidate the cache immediately, + but any external operations (done from other Enso instances) will not be + visible until a cached value expires. Thus if the workflow is expected to + co-operate with other workflows, it may be useful to decrease the cache TTL + or disable it completely by passing `Nothing`. + + Note that completely disabling the caching will affect performance, as some + generic operations may perform multiple cloud requests. + + Changing the TTL invalidates all existing cache entries, because their + expiration time was calculated using the old TTL. +set_file_cache_ttl (duration : Duration | Nothing) = + CacheSettings.setFileCacheTTL duration + Asset_Cache.invalidate_all + +## PRIVATE + ADVANCED + Returns the current file cache TTL. +get_file_cache_ttl -> Duration | Nothing = CacheSettings.getFileCacheTTL diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Enso_File.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Enso_File.enso index 146649d52a69..9c4e09f150b4 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Enso_File.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Enso_File.enso @@ -10,7 +10,9 @@ import project.Enso_Cloud.Data_Link import project.Enso_Cloud.Enso_User.Enso_User import project.Enso_Cloud.Errors.Enso_Cloud_Error import project.Enso_Cloud.Internal.Enso_Path.Enso_Path +import project.Enso_Cloud.Internal.Enso_File_Write_Strategy import project.Enso_Cloud.Internal.Existing_Enso_Asset.Existing_Enso_Asset +import project.Enso_Cloud.Internal.Existing_Enso_Asset.Asset_Cache import project.Enso_Cloud.Internal.Utils import project.Error.Error import project.Errors.Common.Not_Found @@ -21,12 +23,15 @@ import project.Errors.Time_Error.Time_Error import project.Errors.Unimplemented.Unimplemented import project.Network.HTTP.HTTP import project.Network.HTTP.HTTP_Method.HTTP_Method +import project.Network.HTTP.Request_Error import project.Network.URI.URI import project.Nothing.Nothing +import project.Panic.Panic import project.Runtime import project.Runtime.Context import project.System.Environment import project.System.File.Data_Link_Access.Data_Link_Access +import project.System.File.File import project.System.File.File_Access.File_Access import project.System.File.Generic.File_Like.File_Like import project.System.File.Generic.Writable_File.Writable_File @@ -34,6 +39,7 @@ import project.System.File_Format_Metadata.File_Format_Metadata import project.System.Input_Stream.Input_Stream import project.System.Output_Stream.Output_Stream from project.Data.Boolean import Boolean, False, True +from project.Data.Index_Sub_Range.Index_Sub_Range import Last from project.Data.Text.Extensions import all from project.Enso_Cloud.Public_Utils import get_required_field from project.System.File_Format import Auto_Detect, Bytes, File_Format, Plain_Text_Format @@ -178,8 +184,17 @@ type Enso_File value. The value is returned from this method. with_output_stream : Vector File_Access -> (Output_Stream -> Any ! File_Error) -> Any ! File_Error with_output_stream self (open_options : Vector) action = - _ = [open_options, action] - Unimplemented.throw "Writing to Enso_Files is not currently implemented." + Context.Output.if_enabled disabled_message="Writing to an Enso_File is forbidden as the Output context is disabled." panic=False <| + open_as_data_link = (open_options.contains Data_Link_Access.No_Follow . not) && (Data_Link.is_data_link self) + if open_as_data_link then Data_Link.write_data_link_as_stream self open_options action else + if open_options.contains File_Access.Append then Unimplemented.throw "Enso_File currently does not support appending to a file. Instead you may read it, modify and then write the new contents." else + File_Access.ensure_only_allowed_options "with_output_stream" [File_Access.Write, File_Access.Create_New, File_Access.Truncate_Existing, File_Access.Create, Data_Link_Access.No_Follow] open_options <| + allow_existing = open_options.contains File_Access.Create_New . not + tmp_file = File.create_temporary_file "enso-cloud-write-tmp" + Panic.with_finalizer tmp_file.delete <| + perform_upload self allow_existing <| + result = tmp_file.with_output_stream [File_Access.Write] action + result.if_not_error [tmp_file, result] ## PRIVATE ADVANCED @@ -240,7 +255,6 @@ type Enso_File Enso_Asset_Type.File -> File_Format.handle_format_missing_arguments format <| read_with_format effective_format = metadata = File_Format_Metadata.from self - # TODO maybe avoid fetching asset id twice here? maybe move with_input_stream to asset? or helper? self.with_input_stream [File_Access.Read] (stream-> effective_format.read_stream stream metadata) if format != Auto_Detect then read_with_format format else @@ -292,24 +306,28 @@ type Enso_File # Remove secrets from the list - they are handled separately in `Enso_Secret.list`. assets = list_assets self . filter f-> f.asset_type != Enso_Asset_Type.Secret assets.map asset-> - # TODO we could cache the asset maybe? but for how long should we do that? - Enso_File.Value (self.enso_path.resolve asset.name) + file = Enso_File.Value (self.enso_path.resolve asset.name) + Asset_Cache.update file asset + file ## UNSTABLE GROUP Output Creates a subdirectory in a specified directory. create_directory : Text -> Enso_File create_directory self (name : Text) = + effective_name = if name.ends_with "/" then name.drop (Last 1) else name asset = Existing_Enso_Asset.get_asset_reference_for self if asset.is_directory.not then Error.throw (Illegal_Argument.Error "Only directories can contain subdirectories.") else parent_field = if self.is_current_user_root then [] else - [["parentId", [asset.id]]] - body = JS_Object.from_pairs [["title", name]]+parent_field + [["parentId", asset.id]] + body = JS_Object.from_pairs [["title", effective_name]]+parent_field + created_directory = Enso_File.Value (self.enso_path.resolve name) + Asset_Cache.invalidate created_directory response = Utils.http_request_as_json HTTP_Method.Post Utils.directory_api body - # TODO we could cache the asset maybe? but for how long should we do that? created_asset = Existing_Enso_Asset.from_json response created_asset.if_not_error <| - Enso_File.Value (self.enso_path.resolve name) + Asset_Cache.update created_directory created_asset + created_directory ## UNSTABLE GROUP Output @@ -317,8 +335,11 @@ type Enso_File delete : Nothing delete self = if self.enso_path.is_root then Error.throw (Illegal_Argument.Error "The root directory cannot be deleted.") else asset = Existing_Enso_Asset.get_asset_reference_for self - uri = URI.from asset.asset_uri . add_query_argument "force" "true" + # The `if_not_error` here is a workaround for bug https://github.com/enso-org/enso/issues/9669 + uri = asset.if_not_error <| + URI.from asset.asset_uri . add_query_argument "force" "true" response = Utils.http_request HTTP_Method.Delete uri + if asset.is_directory then Asset_Cache.invalidate_subtree self else Asset_Cache.invalidate self response.if_not_error Nothing ## ICON data_output @@ -350,9 +371,9 @@ type Enso_File destination file already exists. Defaults to `False`. move_to : Writable_File -> Boolean -> Nothing ! File_Error move_to self (destination : Writable_File) (replace_existing : Boolean = False) = - _ = [destination, replace_existing] - Context.Output.if_enabled disabled_message="File moving is forbidden as the Output context is disabled." panic=False <| - Unimplemented.throw "Enso_File.move_to is not implemented" + # TODO we could have a fast path if Cloud will support renaming files + generic_copy self destination.file replace_existing . if_not_error <| + self.delete . if_not_error <| destination.file ## GROUP Operators ICON folder @@ -371,6 +392,11 @@ type Enso_File to_text self -> Text = "Enso_File "+self.path + ## PRIVATE + to_js_object : JS_Object + to_js_object self = + JS_Object.from_pairs [["type", "Enso_File"], ["constructor", "new"], ["path", self.path.to_text]] + ## PRIVATE list_assets (parent : Enso_File) -> Vector Existing_Enso_Asset = Existing_Enso_Asset.get_asset_reference_for parent . list_directory @@ -402,16 +428,53 @@ Enso_Asset_Type.from (that:Text) = case that of ## PRIVATE File_Format_Metadata.from (that:Enso_File) = - # TODO this is just a placeholder, until we implement the proper path - path = Nothing - case that.asset_type of - Enso_Asset_Type.Data_Link -> - File_Format_Metadata.Value path=path name=that.name content_type=Data_Link.data_link_content_type - Enso_Asset_Type.Directory -> - File_Format_Metadata.Value path=path name=that.name extension=(that.extension.catch _->Nothing) - Enso_Asset_Type.File -> - File_Format_Metadata.Value path=path name=that.name extension=(that.extension.catch _->Nothing) - other_asset_type -> Error.throw (Illegal_Argument.Error "`File_Format_Metadata` is not available for: "+other_asset_type.to_text+".") + asset_type = that.asset_type.catch File_Error _->Nothing + if asset_type == Enso_Asset_Type.Data_Link then File_Format_Metadata.Value path=that.path name=that.name content_type=Data_Link.data_link_content_type else + File_Format_Metadata.Value path=that.path name=that.name extension=(that.extension.catch _->Nothing) ## PRIVATE File_Like.from (that : Enso_File) = File_Like.Value that + +## PRIVATE +Writable_File.from (that : Enso_File) = + Writable_File.Value that Enso_File_Write_Strategy.instance + +## PRIVATE +upload_file (local_file : File) (destination : Enso_File) (replace_existing : Boolean) -> Enso_File = + result = perform_upload destination replace_existing [local_file, destination] + result.catch Enso_Cloud_Error error-> + is_source_file_not_found = case error of + Enso_Cloud_Error.Connection_Error cause -> case cause of + request_error : Request_Error -> request_error.error_type == 'java.io.FileNotFoundException' + _ -> False + _ -> False + if is_source_file_not_found then Error.throw (File_Error.Not_Found local_file) else result + +## PRIVATE + `generate_request_body_and_result` should return a pair, + where the first element is the request body and the second element is the result to be returned. + It is executed lazily, only after all pre-conditions are successfully met. +perform_upload (destination : Enso_File) (allow_existing : Boolean) (~generate_request_body_and_result) = + parent_directory = destination.parent + if parent_directory.is_nothing then Error.throw (Illegal_Argument.Error "The root directory cannot be a destination for upload. The destination must be a path to a file.") else + parent_directory_asset = Existing_Enso_Asset.get_asset_reference_for parent_directory + # If the parent directory does not exist, we fail. + parent_directory_asset.if_not_error <| + existing_asset = Existing_Enso_Asset.get_asset_reference_for destination + . catch File_Error _->Nothing + if existing_asset.is_nothing.not && allow_existing.not then Error.throw (File_Error.Already_Exists destination) else + if existing_asset.is_nothing.not && existing_asset.asset_type != Enso_Asset_Type.File then Error.throw (Illegal_Argument.Error "The destination must be a path to a file, not "+existing_asset.asset_type.to_text+".") else + existing_asset_id = existing_asset.if_not_nothing <| existing_asset.id + file_name = destination.name + base_uri = URI.from Utils.files_api + . add_query_argument "parent_directory_id" parent_directory_asset.id + . add_query_argument "file_name" file_name + full_uri = case existing_asset_id of + Nothing -> base_uri + _ -> base_uri . add_query_argument "file_id" existing_asset_id + pair = generate_request_body_and_result + Asset_Cache.invalidate destination + response = Utils.http_request HTTP_Method.Post full_uri pair.first + response.if_not_error <| + Asset_Cache.update destination (Existing_Enso_Asset.from_json response) + pair.second diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Internal/Authentication.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Internal/Authentication.enso index 63433c8fdea3..3ba93eb7ccc6 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Internal/Authentication.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Internal/Authentication.enso @@ -27,6 +27,7 @@ import project.Network.HTTP.Response.Response import project.Network.URI.URI import project.Nothing.Nothing import project.Panic.Panic +import project.Runtime.Context import project.Runtime.Ref.Ref import project.System.Environment import project.System.File.File @@ -135,7 +136,7 @@ type Refresh_Token_Data headers = [Header.content_type "application/x-amz-json-1.1", Header.new "X-Amz-Target" "AWSCognitoIdentityProviderService.InitiateAuth"] auth_parameters = JS_Object.from_pairs [["REFRESH_TOKEN", self.refresh_token], ["DEVICE_KEY", Nothing]] payload = JS_Object.from_pairs [["ClientId", self.client_id], ["AuthFlow", "REFRESH_TOKEN_AUTH"], ["AuthParameters", auth_parameters]] - response = HTTP.post self.refresh_url body=(Request_Body.Json payload) headers=headers + response = Context.Output.with_enabled <| HTTP.post self.refresh_url body=(Request_Body.Json payload) headers=headers . catch HTTP_Error error-> case error of HTTP_Error.Status_Error status _ _ -> # If the status code is 400-499, then most likely the reason is that the session has expired, so we ask the user to log in again. @@ -179,4 +180,6 @@ file_get_required_string_field json prefix field_name = case json of _ -> got_type = Meta.type_of result . to_display_text Panic.throw (Illegal_State.Error prefix+"expected `"+field_name+"` to be a string, but got "+got_type+".") - _ -> Panic.throw (Illegal_State.Error prefix+"expected an object, got "+(Meta.type_of json)) + _ -> + got_type = Meta.type_of json . to_display_text + Panic.throw (Illegal_State.Error prefix+"expected an object, got "+got_type) diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Internal/Enso_File_Write_Strategy.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Internal/Enso_File_Write_Strategy.enso new file mode 100644 index 000000000000..bc87f10f74c9 --- /dev/null +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Internal/Enso_File_Write_Strategy.enso @@ -0,0 +1,34 @@ +private + +import project.Error.Error +import project.Errors.Common.Forbidden_Operation +import project.Errors.Unimplemented.Unimplemented +import project.System.File.Existing_File_Behavior.Existing_File_Behavior +import project.System.File.File +from project.Data.Boolean import Boolean, False, True +from project.Enso_Cloud.Enso_File import Enso_File, upload_file +from project.System.File.Generic.File_Write_Strategy import default_append, default_overwrite, default_raise_error, File_Write_Strategy, generic_remote_write_with_local_file + +## PRIVATE + In the Enso_File, we use the Overwrite strategy for Backup. + That is because, the Cloud keeps versions of the file by itself, + so there is no need to duplicate its work on our own - just overwriting the + file still ensures we have a backup. +instance = + File_Write_Strategy.Value default_overwrite default_append default_raise_error write_backing_up=default_overwrite create_dry_run_file remote_write_with_local_file copy_from_local + + +## PRIVATE +create_dry_run_file file copy_original = + _ = [file, copy_original] + Error.throw (Forbidden_Operation.Error "Currently dry-run is not supported for Enso_File, so writing to an Enso_File is forbidden if the Output context is disabled.") + + +## PRIVATE +remote_write_with_local_file file existing_file_behavior action = + if existing_file_behavior == Existing_File_Behavior.Append then Unimplemented.throw "Enso Cloud does not yet support appending to a file. Instead you may read it, modify and then write the new contents." else + generic_remote_write_with_local_file file existing_file_behavior action + +## PRIVATE +copy_from_local (source : File) (destination : Enso_File) (replace_existing : Boolean) = + upload_file source destination replace_existing diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Internal/Existing_Enso_Asset.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Internal/Existing_Enso_Asset.enso index dd0f762d5409..e2db052ce3c7 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Internal/Existing_Enso_Asset.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Internal/Existing_Enso_Asset.enso @@ -7,6 +7,7 @@ import project.Data.Text.Text_Sub_Range.Text_Sub_Range import project.Data.Time.Date_Time.Date_Time import project.Data.Time.Date_Time_Formatter.Date_Time_Formatter import project.Data.Vector.Vector +import project.Enso_Cloud.Cloud_Caching_Settings import project.Enso_Cloud.Enso_User.Enso_User import project.Enso_Cloud.Enso_File.Enso_Asset_Type import project.Enso_Cloud.Enso_File.Enso_File @@ -73,9 +74,7 @@ type Existing_Enso_Asset Fetches the basic information about an existing file from the Cloud. It will fail if the file does not exist. get_asset_reference_for (file : Enso_File) -> Existing_Enso_Asset ! File_Error = - # TODO remove workaround for bug https://github.com/enso-org/cloud-v2/issues/1173 - path = if file.enso_path.is_root then file.enso_path.to_text + "/" else file.enso_path.to_text - Existing_Enso_Asset.resolve_path path if_not_found=(Error.throw (File_Error.Not_Found file)) + fetch_asset_reference file ## PRIVATE Resolves a path to an existing asset in the cloud. @@ -111,3 +110,31 @@ type Existing_Enso_Asset #org = json.get "organizationId" "" asset_type = (id.take (Text_Sub_Range.Before "-")):Enso_Asset_Type Existing_Enso_Asset.Value title id asset_type + + +## PRIVATE +type Asset_Cache + asset_prefix = "asset:" + + asset_key (file : Enso_File) -> Text = Asset_Cache.asset_prefix+file.enso_path.to_text + + ## PRIVATE + invalidate (file : Enso_File) = + Utils.invalidate_cache (Asset_Cache.asset_key file) + + invalidate_subtree (file : Enso_File) = + Utils.invalidate_caches_with_prefix (Asset_Cache.asset_key file) + + invalidate_all = + Utils.invalidate_caches_with_prefix Asset_Cache.asset_prefix + + update (file : Enso_File) (asset : Existing_Enso_Asset) = + Utils.set_cached (Asset_Cache.asset_key file) asset cache_duration=Cloud_Caching_Settings.get_file_cache_ttl + +## PRIVATE + Returns the cached reference or fetches it from the cloud. +fetch_asset_reference (file : Enso_File) -> Existing_Enso_Asset ! File_Error = + # TODO remove workaround for bug https://github.com/enso-org/cloud-v2/issues/1173 + path = if file.enso_path.is_root then file.enso_path.to_text + "/" else file.enso_path.to_text + Utils.get_cached (Asset_Cache.asset_key file) cache_duration=Cloud_Caching_Settings.get_file_cache_ttl <| + Existing_Enso_Asset.resolve_path path if_not_found=(Error.throw (File_Error.Not_Found file)) diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Internal/Utils.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Internal/Utils.enso index f53ec2a96ab1..d9aa760585cc 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Internal/Utils.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Internal/Utils.enso @@ -87,9 +87,10 @@ http_request_as_json (method : HTTP_Method) (url : URI) (body : Request_Body = R Custom error handlers can be provided as a mapping from error codes (defined in the cloud project) to functions that take the full JSON payload and return a custom error. -http_request (method : HTTP_Method) (url : URI) (body : Request_Body = Request_Body.Empty) (additional_headers : Vector = []) (error_handlers : Map Text (Any -> Any) = Map.empty) (retries : Integer = 3) -> Response ! Enso_Cloud_Error = +http_request (method : HTTP_Method) (url : URI) (body : Request_Body = Request_Body.Empty) (additional_headers : Vector = []) (error_handlers : Map Text (Any -> Any) = Map.empty) (retries : Integer = 3) -> Response ! Enso_Cloud_Error = method.if_not_error <| url.if_not_error <| body.if_not_error <| additional_headers.if_not_error <| all_headers = [authorization_header] + additional_headers as_connection_error err = Error.throw (Enso_Cloud_Error.Connection_Error err) + response = HTTP.new.request (Request.new method url headers=all_headers body=body) error_on_failure_code=False . catch HTTP_Error as_connection_error . catch Request_Error as_connection_error @@ -115,5 +116,21 @@ http_request (method : HTTP_Method) (url : URI) (body : Request_Body = Request_B ## PRIVATE Returns the cached value for the given key, or computes it using the given action and caches it for future use. -get_cached (key : Text) ~action (cache_duration : Duration = Duration.new seconds=60) = + If `cache_duration` is set to `Nothing`, then the cache is always skipped. +get_cached (key : Text) ~action (cache_duration : Duration | Nothing = Duration.new seconds=60) = CloudRequestCache.getOrCompute key (_->action) cache_duration + +## PRIVATE + Invalidates the cache entry for the given key. +invalidate_cache (key : Text) = + CloudRequestCache.invalidateEntry key + +## PRIVATE + Invalidates all cache entries that share a common prefix. +invalidate_caches_with_prefix (prefix : Text) = + CloudRequestCache.invalidatePrefix prefix + +## PRIVATE + If `cache_duration` is set to `Nothing`, then this action does not do anything. +set_cached (key : Text) value (cache_duration : Duration | Nothing = Duration.new seconds=60) = + CloudRequestCache.put key value cache_duration diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/System/File_Format.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/System/File_Format.enso index dc587402f425..b804333eb799 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/System/File_Format.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/System/File_Format.enso @@ -50,8 +50,9 @@ type Auto_Detect ## PRIVATE Implements the `File.read` for this `File_Format` read : File -> Problem_Behavior -> Any ! File_Error - read self file on_problems = if file.is_directory then file.list else - reader = Auto_Detect.get_reading_format file + read self file on_problems:Problem_Behavior = if file.is_directory then file.list else + metadata = File_Format_Metadata.from file + reader = Auto_Detect.get_reading_format metadata if reader == Nothing then Error.throw (File_Error.Unsupported_Type file) else reader.read file on_problems diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/CacheSettings.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/CacheSettings.java new file mode 100644 index 000000000000..4fc38f6c3f4a --- /dev/null +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/CacheSettings.java @@ -0,0 +1,15 @@ +package org.enso.base.enso_cloud; + +import java.time.Duration; + +public final class CacheSettings { + private static Duration fileCacheTTL = Duration.ofSeconds(60); + + public static Duration getFileCacheTTL() { + return fileCacheTTL; + } + + public static void setFileCacheTTL(Duration fileCacheTTL) { + CacheSettings.fileCacheTTL = fileCacheTTL; + } +} diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/CloudRequestCache.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/CloudRequestCache.java index 62d8a85f1cc5..3931ff214451 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/CloudRequestCache.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/CloudRequestCache.java @@ -20,15 +20,37 @@ public static void clear() { } public static Object getOrCompute(String key, Function compute, Duration ttl) { + if (ttl == null) { + // If the TTL is null, we deliberately ignore the cache. + return compute.apply(key); + } + var entry = cache.get(key); if (entry != null && entry.expiresAt.isAfter(LocalDateTime.now())) { return entry.value; } else { var value = compute.apply(key); - cache.put(key, new CacheEntry(value, LocalDateTime.now().plus(ttl))); + put(key, value, ttl); return value; } } + public static void invalidateEntry(String key) { + cache.remove(key); + } + + public static void invalidatePrefix(String prefix) { + cache.keySet().removeIf(key -> key.startsWith(prefix)); + } + + public static void put(String key, Object value, Duration ttl) { + if (ttl == null) { + // If the TTL is null, we deliberately ignore the cache. + return; + } + + cache.put(key, new CacheEntry(value, LocalDateTime.now().plus(ttl))); + } + private record CacheEntry(Object value, LocalDateTime expiresAt) {} } diff --git a/test/AWS_Tests/src/Inter_Backend_File_Operations_Spec.enso b/test/AWS_Tests/src/Inter_Backend_File_Operations_Spec.enso index cedd97d24bb2..25e0930f08b1 100644 --- a/test/AWS_Tests/src/Inter_Backend_File_Operations_Spec.enso +++ b/test/AWS_Tests/src/Inter_Backend_File_Operations_Spec.enso @@ -7,6 +7,7 @@ from Standard.Base import all import Standard.Base.Enso_Cloud.Data_Link.Data_Link_Format +import Standard.Base.Errors.Common.Not_Found import Standard.Base.Errors.File_Error.File_Error import Standard.Base.Errors.Illegal_Argument.Illegal_Argument @@ -14,21 +15,94 @@ from Standard.AWS import S3_File from Standard.Test import all +import enso_dev.Base_Tests.Network.Enso_Cloud.Cloud_Tests_Setup.Cloud_Tests_Setup + from project.S3_Spec import api_pending, writable_root, with_default_credentials +type Temporary_Local_File + Value ~get + + make name = Temporary_Local_File.Value <| + File.create_temporary_file "local" name + + cleanup self = + Panic.rethrow <| self.get.delete_if_exists + +type Temporary_S3_File + # Does not have to be lazy, because merely allocating the path does not create anything + Value get + + make location name = Temporary_S3_File.Value <| + location / ("s3-"+name) + + cleanup self = + Panic.rethrow <| self.get.delete_if_exists + +type Temporary_Enso_Cloud_File + Value ~get + + make tmp_dir_name name = Temporary_Enso_Cloud_File.Value <| + location = Enso_File.root / tmp_dir_name + Panic.rethrow <| if location.exists.not then location.parent.create_directory location.name + location / ("cloud-"+name) + + cleanup self = + parent = self.get.parent + Panic.rethrow <| self.get.delete_if_exists + Panic.rethrow <| + needs_delete = parent.list.is_empty . catch File_Error _->False + if needs_delete then parent.delete_if_exists + +type Backend + Ready name:Text make_temp_file + Pending name:Text reason:Text + + is_pending self = case self of + Backend.Pending _ _ -> True + _ -> False + +## Returns the first pending reason from the given vector of values, or Nothing if all values are available. +any_pending (backends : Vector Backend) -> Text|Nothing = + backends.find .is_pending . reason . catch Not_Found _->Nothing + +prepare_available_backends s3_root tmp_dir_name = + builder = Vector.new_builder + + # Local + builder.append (Backend.Ready "Local" Temporary_Local_File.make) + + # Cloud + cloud_setup = Cloud_Tests_Setup.prepare + builder.append <| Panic.rethrow <| case cloud_setup.real_cloud_pending of + Nothing -> Backend.Ready "Enso Cloud" (Temporary_Enso_Cloud_File.make tmp_dir_name) + reason -> Backend.Pending "Enso Cloud" reason + + # S3 + builder.append <| Panic.rethrow <| case api_pending of + Nothing -> Backend.Ready "S3" (Temporary_S3_File.make s3_root) + reason -> Backend.Pending "S3" reason + + builder.to_vector + add_specs suite_builder = - my_writable_s3_dir = writable_root / "inter-backend-test-run-"+(Date_Time.now.format "yyyy-MM-dd_HHmmss.fV" . replace "/" "|")+"/" - sources = [my_writable_s3_dir / "source1.txt", File.create_temporary_file "source2" ".txt"] - destinations = [my_writable_s3_dir / "destination1.txt", File.create_temporary_file "destination2" ".txt"] - sources.each source_file-> destinations.each destination_file-> if source_file.is_a File && destination_file.is_a File then Nothing else - src_typ = Meta.type_of source_file . to_display_text - dest_typ = Meta.type_of destination_file . to_display_text - suite_builder.group "("+src_typ+" -> "+dest_typ+") copying/moving" pending=api_pending group_builder-> + tmp_dir_name = "inter-backend-test-run-"+(Date_Time.now.format "yyyy-MM-dd_HHmmss.fV" . replace "/" "|")+"/" + my_writable_s3_dir = writable_root / tmp_dir_name + + backends = prepare_available_backends my_writable_s3_dir tmp_dir_name + ## TODO: as the number of backends grows, we probably don't want to test every combination anymore. + Still, every backend should have at least X->Local, Local->X, X->X (within itself) and one test with another remote backend (e.g. X->S3). + backends.each source_backend-> backends.each destination_backend-> + suite_builder.group "("+source_backend.name+" -> "+destination_backend.name+") copying/moving" pending=(any_pending [source_backend, destination_backend]) group_builder-> + source_file_provider = source_backend.make_temp_file "src.txt" + destination_file_provider = destination_backend.make_temp_file "dest.txt" group_builder.teardown <| - source_file.delete_if_exists - destination_file.delete_if_exists + source_file_provider.cleanup + destination_file_provider.cleanup group_builder.specify "should be able to copy files" <| + source_file = source_file_provider.get + destination_file = destination_file_provider.get + "Hello".write source_file on_existing_file=Existing_File_Behavior.Overwrite . should_succeed destination_file.delete_if_exists @@ -37,6 +111,9 @@ add_specs suite_builder = source_file.exists . should_be_true group_builder.specify "should be able to move files" <| + source_file = source_file_provider.get + destination_file = destination_file_provider.get + "Hello".write source_file on_existing_file=Existing_File_Behavior.Overwrite . should_succeed destination_file.delete_if_exists @@ -45,6 +122,9 @@ add_specs suite_builder = source_file.exists . should_be_false group_builder.specify "should fail if the source file does not exist" <| + source_file = source_file_provider.get + destination_file = destination_file_provider.get + source_file.delete_if_exists destination_file.delete_if_exists @@ -59,6 +139,9 @@ add_specs suite_builder = destination_file.exists . should_be_false group_builder.specify "should fail to copy/move a file if it exists and replace_existing=False" <| + source_file = source_file_provider.get + destination_file = destination_file_provider.get + "Hello".write source_file on_existing_file=Existing_File_Behavior.Overwrite . should_succeed "World".write destination_file on_existing_file=Existing_File_Behavior.Overwrite . should_succeed @@ -73,6 +156,9 @@ add_specs suite_builder = destination_file.read . should_equal "World" group_builder.specify "should overwrite existing destination in copy/move if replace_existing=True" <| + source_file = source_file_provider.get + destination_file = destination_file_provider.get + "Hello".write source_file on_existing_file=Existing_File_Behavior.Overwrite . should_succeed "World".write destination_file on_existing_file=Existing_File_Behavior.Overwrite . should_succeed @@ -87,21 +173,26 @@ add_specs suite_builder = sample_data_link_content = Data_Link_Format.read_raw_config (enso_project.data / "simple.datalink") # TODO Enso_File datalink once Enso_File & cloud datalink write is supported - data_link_sources = [my_writable_s3_dir / "s3.datalink", File.create_temporary_file "local" ".datalink"] - data_link_destinations = [my_writable_s3_dir / "s3-target.datalink", File.create_temporary_file "local-target" ".datalink"] + datalink_backends = backends.filter b-> b.name != "Enso Cloud" ## This introduces a lot of combinations for testing the datalink copy/move logic, but unfortunately it is needed, because various combinations of backends may rely on different logic (different operations happen under the hood if a file is moved locally vs if it is moved from a local filesystem to S3 or vice versa), and all that different logic may be prone to mis-handling datalinks - so we need to test all paths to ensure coverage. - data_link_sources.each source_file-> data_link_destinations.each destination_file-> - src_typ = Meta.type_of source_file . to_display_text - dest_typ = Meta.type_of destination_file . to_display_text - suite_builder.group "("+src_typ+" -> "+dest_typ+") Data Link copying/moving" pending=api_pending group_builder-> + datalink_backends.each source_backend-> datalink_backends.each destination_backend-> + ## All Data Link tests depend on S3 - even if the backends do not use S3, the datalink itself targets S3, + so `api_pending` is always checked and the test will not be run without S3 config present. + pending = any_pending [source_backend, destination_backend] . if_nothing api_pending + suite_builder.group "("+source_backend.name+" -> "+destination_backend.name+") Data Link copying/moving" pending=pending group_builder-> + source_file_provider = source_backend.make_temp_file "src.datalink" + destination_file_provider = destination_backend.make_temp_file "dest.datalink" group_builder.teardown <| - source_file.delete_if_exists - destination_file.delete_if_exists + source_file_provider.cleanup + destination_file_provider.cleanup group_builder.specify "should be able to copy a datalink file to a regular file" <| with_default_credentials <| + source_file = source_file_provider.get + destination_file = destination_file_provider.get + regular_destination_file = destination_file.parent / destination_file.name+".txt" regular_destination_file.delete_if_exists Data_Link_Format.write_raw_config source_file sample_data_link_content replace_existing=True . should_succeed @@ -112,6 +203,9 @@ add_specs suite_builder = regular_destination_file.read . should_contain '"libraryName": "Standard.AWS"' group_builder.specify "should be able to copy a datalink file to another datalink" <| with_default_credentials <| + source_file = source_file_provider.get + destination_file = destination_file_provider.get + destination_file.delete_if_exists Data_Link_Format.write_raw_config source_file sample_data_link_content replace_existing=True . should_succeed @@ -123,6 +217,9 @@ add_specs suite_builder = Data_Link_Format.read_raw_config destination_file . should_equal sample_data_link_content group_builder.specify "should be able to move a datalink" <| with_default_credentials <| + source_file = source_file_provider.get + destination_file = destination_file_provider.get + destination_file.delete_if_exists Data_Link_Format.write_raw_config source_file sample_data_link_content replace_existing=True . should_succeed @@ -133,6 +230,8 @@ add_specs suite_builder = Data_Link_Format.read_raw_config destination_file . should_equal sample_data_link_content group_builder.specify "should be able to move a regular file to become a datalink" <| with_default_credentials <| + source_file = source_file_provider.get + destination_file = destination_file_provider.get destination_file.delete_if_exists regular_source_file = source_file.parent / source_file.name+".txt" diff --git a/test/AWS_Tests/src/S3_Spec.enso b/test/AWS_Tests/src/S3_Spec.enso index 8a713369a9c4..313951f3d3f8 100644 --- a/test/AWS_Tests/src/S3_Spec.enso +++ b/test/AWS_Tests/src/S3_Spec.enso @@ -33,7 +33,7 @@ test_credentials -> AWS_Credential ! Illegal_State = with_default_credentials ~action = AWS_Credential.with_default_override test_credentials action -api_pending = if test_credentials.is_nothing then "No Access Key found." else Nothing +api_pending = if test_credentials.is_error then test_credentials.catch.message else Nothing bucket_name = "enso-data-samples" writable_bucket_name = "enso-ci-s3-test-stage" diff --git a/test/Base_Tests/src/Network/Enso_Cloud/Cloud_Tests_Setup.enso b/test/Base_Tests/src/Network/Enso_Cloud/Cloud_Tests_Setup.enso index a371f266720d..90f75a7e4509 100644 --- a/test/Base_Tests/src/Network/Enso_Cloud/Cloud_Tests_Setup.enso +++ b/test/Base_Tests/src/Network/Enso_Cloud/Cloud_Tests_Setup.enso @@ -1,6 +1,8 @@ from Standard.Base import all import Standard.Base.Errors.Illegal_State.Illegal_State +import Standard.Base.Runtime.Context +from Standard.Test import Problems import Standard.Test.Test_Environment @@ -11,23 +13,24 @@ polyglot java import org.enso.base.enso_cloud.CloudAPI type Cloud_Tests_Setup Mock api_url:URI credentials_location:File - Cloud api_url:URI credentials_location:File + Cloud None - with_prepared_environment self ~action = - Cloud_Tests_Setup.reset - Panic.with_finalizer Cloud_Tests_Setup.reset <| - if self == Cloud_Tests_Setup.None then action else + with_prepared_environment self ~action = case self of + Cloud_Tests_Setup.Mock _ _ -> + Cloud_Tests_Setup.reset + Panic.with_finalizer Cloud_Tests_Setup.reset <| Test_Environment.unsafe_with_environment_override "ENSO_CLOUD_API_URI" self.api_url.to_text <| Test_Environment.unsafe_with_environment_override "ENSO_CLOUD_CREDENTIALS_FILE" self.credentials_location.absolute.normalize.path <| action + _ -> action pending self = case self of Cloud_Tests_Setup.None -> "Cloud tests run only if ENSO_RUN_REAL_CLOUD_TEST or ENSO_HTTP_TEST_HTTPBIN_URL environment variable is defined." _ -> Nothing real_cloud_pending self = case self of - Cloud_Tests_Setup.Cloud _ _ -> Nothing + Cloud_Tests_Setup.Cloud -> Nothing _ -> "These cloud tests only run if ENSO_RUN_REAL_CLOUD_TEST is defined, as they require a proper cloud environment for testing, not just a minimal mock." httpbin_pending self = @@ -72,17 +75,16 @@ type Cloud_Tests_Setup reset = CloudAPI.flushCloudCaches ## Detects the setup based on environment settings. + If `ENSO_RUN_REAL_CLOUD_TEST` is defined, the tests will try running + against the real cloud, relying on the default cloud configuration (which + can be overridden using environment variables). + Otherwise, if `ENSO_HTTP_TEST_HTTPBIN_URL` is defined, the tests are run + against a mock implementation, prepare : Cloud_Tests_Setup prepare = real_cloud = Environment.get "ENSO_RUN_REAL_CLOUD_TEST" . is_nothing . not case real_cloud of - True -> - api_url = Environment.get "ENSO_CLOUD_API_URI" . if_nothing <| - Panic.throw (Illegal_State.Error "If ENSO_RUN_REAL_CLOUD_TEST is defined, ENSO_CLOUD_API_URI must be defined as well.") - credentials_location = Environment.get "ENSO_CLOUD_CREDENTIALS_FILE" . if_nothing <| - Panic.throw (Illegal_State.Error "If ENSO_RUN_REAL_CLOUD_TEST is defined, ENSO_CLOUD_CREDENTIALS_FILE must be defined as well.") - - Cloud_Tests_Setup.Cloud (URI.from api_url) (File.new credentials_location) + True -> Cloud_Tests_Setup.Cloud False -> Cloud_Tests_Setup.prepare_mock_setup ## Runs the action inside of an environment set up with the Cloud Mock @@ -98,6 +100,8 @@ type Cloud_Tests_Setup setup = Cloud_Tests_Setup.prepare_mock_setup custom_credentials setup.with_prepared_environment action + ## Prepares a setup that will always call into the local cloud mock. + This is useful for scenarios like testing credential logic. prepare_mock_setup (custom_credentials : Mock_Credentials | Nothing = Nothing) -> Cloud_Tests_Setup = base_url = Environment.get "ENSO_HTTP_TEST_HTTPBIN_URL" if base_url.is_nothing then Cloud_Tests_Setup.None else @@ -105,8 +109,11 @@ type Cloud_Tests_Setup enso_cloud_url = with_slash + "enso-cloud-mock/" credentials_payload = (custom_credentials.if_nothing (Mock_Credentials.default with_slash)).to_json - tmp_cred_file = File.create_temporary_file "enso-test-credentials" ".json" - credentials_payload.write tmp_cred_file + # We need to override the Context in case the tests were running in disabled context mode + tmp_cred_file = Context.Output.with_enabled <| + tmp_cred_file = File.create_temporary_file "enso-test-credentials" ".json" + credentials_payload.write tmp_cred_file + Problems.assume_no_problems <| tmp_cred_file Cloud_Tests_Setup.Mock (URI.from enso_cloud_url) tmp_cred_file diff --git a/test/Base_Tests/src/Network/Enso_Cloud/Enso_Cloud_Spec.enso b/test/Base_Tests/src/Network/Enso_Cloud/Enso_Cloud_Spec.enso index bbdf7ef1d615..8894d048abd9 100644 --- a/test/Base_Tests/src/Network/Enso_Cloud/Enso_Cloud_Spec.enso +++ b/test/Base_Tests/src/Network/Enso_Cloud/Enso_Cloud_Spec.enso @@ -8,6 +8,7 @@ import Standard.Base.Errors.Illegal_State.Illegal_State from Standard.Test import all import Standard.Test.Test_Environment +from Standard.Test.Execution_Context_Helpers import run_with_and_without_output import project.Network.Enso_Cloud.Cloud_Tests_Setup.Cloud_Tests_Setup @@ -109,7 +110,7 @@ add_specs suite_builder setup:Cloud_Tests_Setup = AuthenticationProvider.getAuthenticationServiceEnsoInstance.auth_data.get.access_token base_credentials = Lazy_Ref.Value <| Mock_Credentials.default Cloud_Tests_Setup.prepare_mock_setup.httpbin_uri - group_builder.specify "refreshes an expired token" <| + group_builder.specify "refreshes an expired token" <| run_with_and_without_output <| Cloud_Tests_Setup.run_with_mock_cloud custom_credentials=base_credentials.get.locally_expired <| previous_token = get_current_token ## Trigger some cloud endpoint - it should succeed @@ -134,7 +135,7 @@ add_specs suite_builder setup:Cloud_Tests_Setup = count_after = mock_setup.get_expired_token_failures_count count_after . should_equal count_before+1 - group_builder.specify "refreshes a token that is about to expire" <| + group_builder.specify "refreshes a token that is about to expire" <| run_with_and_without_output <| # The token here is not yet expired, but we still refresh it. Cloud_Tests_Setup.run_with_mock_cloud custom_credentials=base_credentials.get.about_to_expire <| previous_token = get_current_token diff --git a/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso b/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso index 23e36ab5aced..37cc86a03ae6 100644 --- a/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso +++ b/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso @@ -1,225 +1,317 @@ from Standard.Base import all +import Standard.Base.Enso_Cloud.Cloud_Caching_Settings +import Standard.Base.Errors.Common.Forbidden_Operation import Standard.Base.Errors.Common.Not_Found import Standard.Base.Errors.File_Error.File_Error import Standard.Base.Errors.Illegal_Argument.Illegal_Argument import Standard.Base.Errors.Unimplemented.Unimplemented +import Standard.Base.Runtime.Context from Standard.Test import all import Standard.Test.Test_Environment import project.Network.Enso_Cloud.Cloud_Tests_Setup.Cloud_Tests_Setup -add_specs suite_builder setup:Cloud_Tests_Setup = setup.with_prepared_environment <| - suite_builder.group "Enso Cloud Files" pending=setup.real_cloud_pending group_builder-> - group_builder.specify "should be able to list the root directory" <| - assets = Enso_File.root.list - # We don't a priori know the contents, so we can only check very generic properties - assets . should_be_a Vector - assets.each f-> f.should_be_a Enso_File - - ## We assume that it contains a test file `test_file.json` - TODO in future iterations this file will be created by the test suite itself, to make it self-contained - The file is expected to contain: - [1, 2, 3, "foo"] - assets.map .name . should_contain "test_file.json" - - group_builder.specify "should allow to create and delete a directory" <| - my_name = "my_test_dir-" + (Random.uuid.take 5) - my_dir = Enso_File.root.create_directory my_name - my_dir.should_succeed - delete_on_fail caught_panic = - my_dir.delete - Panic.throw caught_panic - Panic.catch Any handler=delete_on_fail <| Test.with_retries <| - my_dir.is_directory . should_be_true - my_dir.exists . should_be_true - my_dir.name . should_equal my_name - Enso_File.root.list . should_contain my_dir - - my_dir.delete . should_succeed - - Test.with_retries <| - Enso_File.root.list . should_not_contain my_dir - - # TODO the dir still shows as 'existing' after deletion, probably because it still is there in the Trash - # my_dir.exists . should_be_false - - group_builder.specify "should set the current working directory by environment variable" <| - # If nothing set, defaults to root: - Enso_File.current_working_directory . should_equal Enso_File.root - - subdir = Enso_File.root.create_directory "my_test_CWD-"+(Random.uuid.take 5) - subdir.should_succeed - cleanup = +type Temporary_Directory + Value ~get + + make -> Temporary_Directory = Temporary_Directory.Value <| + name = "test-run-Enso_File-"+(Date_Time.now.format "yyyy-MM-dd_HHmmss.fV" . replace "/" "|") + + # Create the expected directory structure. + root = Panic.rethrow <| Enso_File.root.create_directory name + Panic.rethrow <| Temporary_Directory.test_file_text.write (root / "test_file.json") + sub = Panic.rethrow <| root.create_directory "test-directory" + Panic.rethrow <| "Hello Another!".write (sub / "another.txt") + root + + cleanup self = self.get.delete_if_exists + + test_file_text = '[1, 2, 3, "foo"]' + +add_specs suite_builder setup:Cloud_Tests_Setup = suite_builder.group "Enso Cloud Files" pending=setup.real_cloud_pending group_builder-> + test_root = Temporary_Directory.make + group_builder.teardown test_root.cleanup + + group_builder.specify "should be able to list the root directory" <| + assets = Enso_File.root.list + # We don't a priori know the contents, so we can only check very generic properties + assets . should_be_a Vector + assets.each f-> f.should_be_a Enso_File + + # We know something about contents of our prepared test directory though + test_root.get.list.map .name . should_contain "test_file.json" + + group_builder.specify "should allow to create and delete a directory" <| + my_name = "my_test_dir-" + (Random.uuid.take 5) + my_dir = Enso_File.root.create_directory my_name + my_dir.should_succeed + delete_on_fail caught_panic = + my_dir.delete + Panic.throw caught_panic + Panic.catch Any handler=delete_on_fail <| Test.with_retries <| + my_dir.is_directory . should_be_true + my_dir.exists . should_be_true + my_dir.name . should_equal my_name + Enso_File.root.list . should_contain my_dir + + my_dir.delete . should_succeed + + Test.with_retries <| + Enso_File.root.list . should_not_contain my_dir + my_dir.exists . should_be_false + + group_builder.specify "should set the current working directory by environment variable" <| + # If nothing set, defaults to root: + Enso_File.current_working_directory . should_equal Enso_File.root + + subdir = Enso_File.root.create_directory "my_test_CWD-"+(Random.uuid.take 5) + subdir.should_succeed + cleanup = + Enso_User.flush_caches + subdir.delete + Panic.with_finalizer cleanup <| + Test_Environment.unsafe_with_environment_override "ENSO_PROJECT_DIRECTORY_PATH" subdir.path <| + # Flush caches to ensure fresh dir is used Enso_User.flush_caches - subdir.delete - Panic.with_finalizer cleanup <| - Test_Environment.unsafe_with_environment_override "ENSO_PROJECT_DIRECTORY_PATH" subdir.path <| - # Flush caches to ensure fresh dir is used - Enso_User.flush_caches - - Enso_File.current_working_directory . should_equal subdir - - # It should be back to default afterwards: - Enso_File.current_working_directory . should_equal Enso_File.root - - group_builder.specify "should allow to find a file by name" <| - # TODO the file should be created programmatically when write is implemented - f = Enso_File.root / "test_file.json" - f.should_succeed - f.name . should_equal "test_file.json" - f.is_directory . should_be_false + + Enso_File.current_working_directory . should_equal subdir + + # It should be back to default afterwards: + Enso_File.current_working_directory . should_equal Enso_File.root + + group_builder.specify "should allow to find a file by name" <| + f = test_root.get / "test_file.json" + f.should_succeed + f.name . should_equal "test_file.json" + f.is_directory . should_be_false + f.exists . should_be_true + + group_builder.specify "should work if cache is disabled" <| + old_ttl = Cloud_Caching_Settings.get_file_cache_ttl + Panic.with_finalizer (Cloud_Caching_Settings.set_file_cache_ttl old_ttl) <| + # Disable the cache + Cloud_Caching_Settings.set_file_cache_ttl Nothing + + # We cannot easily check if one or two requests are made, but we just ensure that this doesn't crash. + f = test_root.get / "test_file.json" + test_root.get.list . should_contain f + f.exists . should_be_true f.exists . should_be_true - group_builder.specify "should be able to find a file by path" <| - File.new "enso://"+Enso_User.current.organization_name+"/" . should_equal Enso_File.root - File.new "enso://"+Enso_User.current.organization_name+"/test_file.json" . should_equal (Enso_File.root / "test_file.json") - File.new "enso://"+Enso_User.current.organization_name+"/abc/" . should_equal (Enso_File.root / "abc") - - group_builder.specify "should fail to read nonexistent files" <| - f = Enso_File.root / "nonexistent_file.json" - f.should_succeed - f.exists . should_be_false - r = f.read - r.should_fail_with File_Error - r.catch.should_be_a File_Error.Not_Found - - group_builder.specify "should not allow to create a directory inside of a regular file" <| - # TODO the file should be created programmatically when write is implemented - test_file = Enso_File.root / "test_file.json" - test_file.exists . should_be_true - - r = test_file.create_directory "my_test_dir" - r.should_fail_with Illegal_Argument - - group_builder.specify "should delete all contents of a directory when deleting a directory" pending="TODO discuss recursive delete" <| - dir1 = Enso_File.root.create_directory "my_test_dir1"+(Random.uuid.take 5) - dir1.should_succeed - - dir2 = dir1.create_directory "my_test_dir2" - dir2.should_succeed - - dir1.delete . should_succeed - - Test.with_retries <| - dir1.exists . should_be_false - # The inner directory should also have been trashed if its parent is removed - dir2.exists . should_be_false - - group_builder.specify "should not allow to delete the root directory" <| - Enso_File.root.delete . should_fail_with Illegal_Argument - - group_builder.specify "should be able to create and delete a file" pending="TODO: Cloud file write support" <| - Error.throw "TODO" - - expected_file_text = '[1, 2, 3, "foo"]' - group_builder.specify "should be able to read and decode a file using various formats" <| - # TODO the file should be created programmatically when write is implemented - test_file = Enso_File.root / "test_file.json" - test_file.exists . should_be_true - - test_file.read Plain_Text . should_equal expected_file_text - - # auto-detection of JSON format: - json = test_file.read - json.should_be_a Vector - json.should_equal [1, 2, 3, "foo"] - - test_file.read_bytes . should_equal expected_file_text.utf_8 - - group_builder.specify "should be able to read the file by path using Data.read" <| - Data.read "enso://"+Enso_User.current.organization_name+"/test_file.json" . should_equal [1, 2, 3, "foo"] - Data.read "enso://"+Enso_User.current.organization_name+"/test-directory/another.txt" . should_equal "Hello Another!" - - r = Data.read "enso://"+Enso_User.current.organization_name+"/test-directory/nonexistent-directory/some-file.txt" - r.should_fail_with File_Error - r.catch.should_be_a File_Error.Not_Found - - group_builder.specify "should be able to open a file as input stream" <| - test_file = Enso_File.root / "test_file.json" - test_file.exists . should_be_true - - bytes = test_file.with_input_stream [File_Access.Read] stream-> - stream.read_all_bytes - - bytes.should_equal expected_file_text.utf_8 - - group_builder.specify "should be able to read file metadata" <| - Enso_File.root.exists . should_be_true - Enso_File.root.name . should_equal "" - - # TODO this structure should be created once we get file writing - dir = Enso_File.root / "test-directory" - dir.exists.should_be_true - dir.name . should_equal "test-directory" - dir.extension . should_equal "" - dir.is_directory.should_be_true - dir.is_regular_file.should_be_false - dir.size . should_fail_with Illegal_Argument - - nested_file = dir / "another.txt" - nested_file.exists.should_be_true - nested_file.name . should_equal "another.txt" - nested_file.extension . should_equal ".txt" - nested_file.is_directory.should_be_false - nested_file.is_regular_file.should_be_true - nested_file.size . should_equal nested_file.read_bytes.length - nested_file.creation_time . should_be_a Date_Time - nested_file.last_modified_time . should_be_a Date_Time - - Enso_File.root.parent . should_equal Nothing - Enso_File.root.path . should_equal ("enso://"+Enso_User.current.organization_name+"/") - - dir.path . should_equal "enso://"+Enso_User.current.organization_name+"/test-directory" - dir.parent . should_equal Enso_File.root - dir.is_descendant_of Enso_File.root . should_be_true - Enso_File.root.is_descendant_of dir . should_be_false - - nested_file.path . should_equal "enso://"+Enso_User.current.organization_name+"/test-directory/another.txt" - nested_file.parent . should_equal dir - - nested_file.is_descendant_of dir . should_be_true - nested_file.is_descendant_of Enso_File.root . should_be_true - - # Some edge cases - (Enso_File.root / "test-directory-longer-name") . is_descendant_of (Enso_File.root / "test-directory") . should_be_false - (Enso_File.root / "test-directory" / "non-existent") . is_descendant_of (Enso_File.root / "test-directory") . should_be_true - (Enso_File.root / "test-directory") . is_descendant_of (Enso_File.root / "test-directory" / "non-existent") . should_be_false - - group_builder.specify "allows / as well as .. and . in resolve" <| - (Enso_File.root / "a/b/c") . should_equal (Enso_File.root / "a" / "b" / "c") - (Enso_File.root / "a///b/c") . should_equal (Enso_File.root / "a" / "b" / "c") - (Enso_File.root / "a/b/c/./././d/e/../f/../..") . should_equal (Enso_File.root / "a" / "b" / "c") - - r = Enso_File.root / ".." - r.should_fail_with Illegal_Argument - r.catch.to_display_text . should_contain "Cannot move above root" - - group_builder.specify "currently does not support metadata for directories" <| - # TODO this test should be 'reversed' and merged with above once the metadata is implemented - dir = Enso_File.root / "test-directory" - Test.expect_panic Unimplemented dir.creation_time - Test.expect_panic Unimplemented dir.last_modified_time - - group_builder.specify "should be able to read other file metadata" pending="TODO needs further design" <| - nested_file = Enso_File.root / "test-directory" / "another.txt" - - nested_file.is_absolute.should_be_true - nested_file.absolute . should_equal nested_file - nested_file.normalize . should_equal nested_file - nested_file.posix_permissions . should_be_a File_Permissions - nested_file.is_writable . should_be_a Boolean - - group_builder.specify "should be able to copy a file" pending="TODO Cloud file writing" <| - nested_file = Enso_File.root / "test-directory" / "another.txt" - new_file = nested_file.copy_to "TODO" - nested_file.exists.should_be_true - new_file.exists.should_be_true - - group_builder.specify "should be able to move a file" pending="TODO Cloud file writing" <| - nested_file = Enso_File.root / "test-directory" / "another.txt" - nested_file.move_to "TODO" - nested_file.exists . should_be_false + Test.with_clue "metadata is also readable if the Output Context is disabled: " <| + Context.Output.with_disabled <| + f.size.should_equal 16 + + group_builder.specify "should be able to find a file by path" <| + File.new "enso://"+Enso_User.current.organization_name+"/" . should_equal Enso_File.root + File.new "enso://"+Enso_User.current.organization_name+"/test_file.json" . should_equal (Enso_File.root / "test_file.json") + File.new "enso://"+Enso_User.current.organization_name+"/abc/" . should_equal (Enso_File.root / "abc") + + group_builder.specify "should fail to read nonexistent files" <| + f = Enso_File.root / "nonexistent_file.json" + f.should_succeed + f.exists . should_be_false + r = f.read + r.should_fail_with File_Error + r.catch.should_be_a File_Error.Not_Found + + group_builder.specify "should not allow to create a directory inside of a regular file" <| + test_file = test_root.get / "test_file.json" + test_file.exists . should_be_true + + r = test_file.create_directory "my_test_dir" + r.should_fail_with Illegal_Argument + + # TODO expand with #8993 + group_builder.specify "should delete all contents of a directory when deleting a directory" <| + dir1 = test_root.get.create_directory "my_test_dir1"+(Random.uuid.take 5) + dir1.should_succeed + + dir2 = dir1.create_directory "my_test_dir2" + dir2.should_succeed + + dir1.delete . should_succeed + + Test.with_retries <| + dir1.exists . should_be_false + # The inner directory should also have been trashed if its parent is removed + dir2.exists . should_be_false + + group_builder.specify "should not allow to delete the root directory" <| + Enso_File.root.delete . should_fail_with Illegal_Argument + + # See Inter_Backend_File_Operations_Spec for copy/move tests + group_builder.specify "should be able to write a file using with_output_stream" <| + f = test_root.get / "written_file.txt" + r = f.with_output_stream [File_Access.Write] output_stream-> + output_stream.write_bytes "Hello".utf_8 + 42 + r.should_equal 42 + f.read Plain_Text . should_equal "Hello" + + group_builder.specify "will respect Create_New in with_output_stream" <| + test_file = test_root.get / "test_file.json" + test_file.exists . should_be_true + + r = test_file.with_output_stream [File_Access.Create_New, File_Access.Write] output_stream-> + output_stream.write_bytes "ABC".utf_8 + 42 + r.should_fail_with File_Error + r.catch.should_be_a File_Error.Already_Exists + test_file.read Plain_Text . should_equal Temporary_Directory.test_file_text + + group_builder.specify "should be able to write a file using write_bytes" <| + f = test_root.get / "written_file2.txt" + "hi!".utf_8.write_bytes f . should_succeed + f.read Plain_Text . should_equal "hi!" + + group_builder.specify "does not currently support append" <| + f = test_root.get / "written_file3.txt" + Test.expect_panic Unimplemented <| + f.with_output_stream [File_Access.Append, File_Access.Write] output_stream-> + output_stream.write_bytes "DEF".utf_8 + 42 + + group_builder.specify "does not create additional files in Backup mode, because Cloud has its own versioning" <| + dir = test_root.get.create_directory "empty-folder" + dir.list.should_equal [] + + f = dir / "file.txt" + f.exists.should_be_false + "ABC".write f on_existing_file=Existing_File_Behavior.Overwrite . should_equal f + "DEF".write f on_existing_file=Existing_File_Behavior.Backup . should_equal f + f.read Plain_Text . should_equal "DEF" + + # But there should not be any other files in the directory + dir.list.should_equal [f] + + group_builder.specify "fails to write with Existing_File_Behavior.Error if the file exists" <| + f = test_root.get / "existing-file-behavior-error.txt" + + f.exists.should_be_false + "ABC".write f on_existing_file=Existing_File_Behavior.Error . should_equal f + r = "DEF".write f on_existing_file=Existing_File_Behavior.Error + r.should_fail_with File_Error + r.catch.should_be_a File_Error.Already_Exists + + # The file contents stay unchanged - write was prevented + f.read . should_equal "ABC" + + group_builder.specify "fails to write if Output context is disabled" <| + f = test_root.get / "output-disabled.txt" + f.exists . should_be_false + + Context.Output.with_disabled <| + r = "ABC".write f + r.should_fail_with Forbidden_Operation + r.catch.to_display_text . should_contain "Currently dry-run is not supported for Enso_File" + # The file should not have been created + f.exists . should_be_false + + group_builder.specify "fails to write if the parent directory does not exist" <| + f = test_root.get / "nonexistent-dir" / "file.txt" + r = "ABC".write f + r.should_fail_with File_Error + r.catch.should_be_a File_Error.Not_Found + + group_builder.specify "should be able to read and decode a file using various formats" <| + test_file = test_root.get / "test_file.json" + test_file.exists . should_be_true + + test_file.read Plain_Text . should_equal Temporary_Directory.test_file_text + + # auto-detection of JSON format: + json = test_file.read + json.should_be_a Vector + json.should_equal [1, 2, 3, "foo"] + + test_file.read_bytes . should_equal Temporary_Directory.test_file_text.utf_8 + + group_builder.specify "should be able to read the file by path using Data.read" <| + test_root.get.path . should_contain "enso://" + Data.read test_root.get.path+"/test_file.json" . should_equal [1, 2, 3, "foo"] + Data.read test_root.get.path+"/test-directory/another.txt" . should_equal "Hello Another!" + + r = Data.read test_root.get.path+"/test-directory/nonexistent-directory/some-file.txt" + r.should_fail_with File_Error + r.catch.should_be_a File_Error.Not_Found + + group_builder.specify "should be able to open a file as input stream" <| + test_file = test_root.get / "test_file.json" + test_file.exists . should_be_true + + bytes = test_file.with_input_stream [File_Access.Read] stream-> + stream.read_all_bytes + + bytes.should_equal Temporary_Directory.test_file_text.utf_8 + + group_builder.specify "should be able to read file metadata" <| + Enso_File.root.exists . should_be_true + Enso_File.root.name . should_equal "" + + dir = test_root.get / "test-directory" + dir.exists.should_be_true + dir.name . should_equal "test-directory" + dir.extension . should_equal "" + dir.is_directory.should_be_true + dir.is_regular_file.should_be_false + dir.size . should_fail_with Illegal_Argument + + nested_file = dir / "another.txt" + nested_file.exists.should_be_true + nested_file.name . should_equal "another.txt" + nested_file.extension . should_equal ".txt" + nested_file.is_directory.should_be_false + nested_file.is_regular_file.should_be_true + nested_file.size . should_equal nested_file.read_bytes.length + nested_file.creation_time . should_be_a Date_Time + nested_file.last_modified_time . should_be_a Date_Time + + Enso_File.root.parent . should_equal Nothing + Enso_File.root.path . should_equal ("enso://"+Enso_User.current.organization_name+"/") + + dir.path . should_contain "enso://" + dir.path . should_contain "/test-directory" + dir.parent . should_equal test_root.get + dir.is_descendant_of Enso_File.root . should_be_true + Enso_File.root.is_descendant_of dir . should_be_false + + nested_file.path . should_contain "enso://" + nested_file.path . should_contain "/test-directory/another.txt" + nested_file.parent . should_equal dir + + nested_file.is_descendant_of dir . should_be_true + nested_file.is_descendant_of Enso_File.root . should_be_true + + # Some edge cases + (Enso_File.root / "test-directory-longer-name") . is_descendant_of (Enso_File.root / "test-directory") . should_be_false + (Enso_File.root / "test-directory" / "non-existent") . is_descendant_of (Enso_File.root / "test-directory") . should_be_true + (Enso_File.root / "test-directory") . is_descendant_of (Enso_File.root / "test-directory" / "non-existent") . should_be_false + + group_builder.specify "allows / as well as .. and . in resolve" <| + (Enso_File.root / "a/b/c") . should_equal (Enso_File.root / "a" / "b" / "c") + (Enso_File.root / "a///b/c") . should_equal (Enso_File.root / "a" / "b" / "c") + (Enso_File.root / "a/b/c/./././d/e/../f/../..") . should_equal (Enso_File.root / "a" / "b" / "c") + + r = Enso_File.root / ".." + r.should_fail_with Illegal_Argument + r.catch.to_display_text . should_contain "Cannot move above root" + + group_builder.specify "currently does not support metadata for directories" <| + # TODO this test should be 'reversed' and merged with above once the metadata is implemented + dir = test_root.get / "test-directory" + Test.expect_panic Unimplemented dir.creation_time + Test.expect_panic Unimplemented dir.last_modified_time + + group_builder.specify "should be able to read other file metadata" pending="TODO needs further design" <| + nested_file = Enso_File.root / "test-directory" / "another.txt" + + nested_file.is_absolute.should_be_true + nested_file.absolute . should_equal nested_file + nested_file.normalize . should_equal nested_file + nested_file.posix_permissions . should_be_a File_Permissions + nested_file.is_writable . should_be_a Boolean main filter=Nothing = setup = Cloud_Tests_Setup.prepare diff --git a/test/Table_Tests/src/IO/Cloud_Spec.enso b/test/Table_Tests/src/IO/Cloud_Spec.enso new file mode 100644 index 000000000000..a0e47c7bede3 --- /dev/null +++ b/test/Table_Tests/src/IO/Cloud_Spec.enso @@ -0,0 +1,28 @@ +from Standard.Base import all + +from Standard.Table import all + +from Standard.Test import all + +import enso_dev.Base_Tests.Network.Enso_Cloud.Cloud_Tests_Setup.Cloud_Tests_Setup + +import project.Util + +main filter=Nothing = + suite = Test.build suite_builder-> + add_specs suite_builder + suite.run_with_filter filter + + +add_specs suite_builder = + cloud_setup = Cloud_Tests_Setup.prepare + suite_builder.group "IO operations on Enso Cloud files" pending=cloud_setup.real_cloud_pending group_builder-> + group_builder.specify "writing Excel" <| + t = Table.new [["X", [1, 2, 3]], ["Y", ["a", "b", "c"]]] + + f = Enso_File.root / "write-test-"+(Date_Time.now.format "yyyy-MM-dd_HHmmss.fV" . replace "/" "|")+".xlsx" + t.write f . should_equal f + Panic.with_finalizer f.delete_if_exists <| + workbook = f.read + workbook.should_be_a Excel_Workbook + workbook.read "EnsoSheet" . should_equal t diff --git a/test/Table_Tests/src/IO/Main.enso b/test/Table_Tests/src/IO/Main.enso index d632c433d3ba..6bf28f1f0189 100644 --- a/test/Table_Tests/src/IO/Main.enso +++ b/test/Table_Tests/src/IO/Main.enso @@ -2,6 +2,7 @@ from Standard.Base import all from Standard.Test import all +import project.IO.Cloud_Spec import project.IO.Csv_Spec import project.IO.Data_Link_Formats_Spec import project.IO.Delimited_Read_Spec @@ -12,6 +13,7 @@ import project.IO.Formats_Spec import project.IO.Json_Spec add_specs suite_builder = + Cloud_Spec.add_specs suite_builder Csv_Spec.add_specs suite_builder Delimited_Read_Spec.add_specs suite_builder Delimited_Write_Spec.add_specs suite_builder