From add3f527daae249a51c7af92a6290ddf005105d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Fri, 19 Jan 2024 13:58:58 +0100 Subject: [PATCH 01/14] checkpoint --- .../0.0.0-dev/src/Data/Enso_Cloud/Enso_File.enso | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Enso_File.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Enso_File.enso index a394df9a4c1f..26d26fd582df 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Enso_File.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Enso_File.enso @@ -33,6 +33,18 @@ type Enso_File root : Enso_File root = Enso_File.Value "" "" "" Enso_Asset_Type.Directory + ## Represents the current working directory. + + If the workflow is running on the Cloud, this will be the directory + containing the current project. + + If the workflow is running locally, this will default to the root + directory. + current_working_directory : Enso_File + current_working_directory = + # TODO + Enso_File.root + ## PRIVATE Target URI for the api internal_uri : Text From 7ee48c9bc3908e467de9d23487b362b529da2fac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Fri, 19 Jan 2024 14:03:02 +0100 Subject: [PATCH 02/14] checkpoint 2 --- .../0.0.0-dev/src/Data/Enso_Cloud/Enso_Secret.enso | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Enso_Secret.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Enso_Secret.enso index 1dc37df9bb88..e6e55accc10a 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Enso_Secret.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Enso_Secret.enso @@ -32,14 +32,15 @@ type Enso_Secret - name: The name of the secret - value: The value of the secret - parent: The parent folder for the secret. If `Nothing` then it will be - created in the root folder. + created in the current working directory. create : Text -> Text -> Enso_File | Nothing -> Enso_Secret create name:Text value:Text parent:(Enso_File | Nothing)=Nothing = if name == "" then Error.throw (Illegal_Argument.Error "Secret name cannot be empty") else if Context.Output.is_enabled.not then Error.throw (Forbidden_Operation.Error "Creating a secret is forbidden as the Output context is disabled.") else if name.starts_with "connection-" then Error.throw (Illegal_Argument.Error "Secret name cannot start with 'connection-'") else if Enso_Secret.exists name parent then Error.throw (Illegal_Argument.Error "Secret with this name already exists.") else auth_header = Utils.authorization_header - parent_id_pair = if parent.is_nothing then [] else [["parentDirectoryId", parent.id]] + parent_id = (parent.if_nothing Enso_File.current_working_directory).id + parent_id_pair = if parent_id == "" then [] else [["parentDirectoryId", parent.id]] body = JS_Object.from_pairs [["name", name], ["value", value]]+parent_id_pair response = HTTP.post Utils.secrets_api body HTTP_Method.Post [auth_header] id = response.decode_as_json @@ -60,10 +61,10 @@ type Enso_Secret Arguments: - folder: The folder to get the secrets from. If `Nothing` then will get - the secrets from the root folder. + the secrets from the current working directory. list : Enso_File | Nothing -> Vector Enso_Secret list parent:(Enso_File | Nothing)=Nothing = - secrets_as_files = list_assets (parent.if_nothing Enso_File.root) . filter f-> f.asset_type == Enso_Asset_Type.Secret + secrets_as_files = list_assets (parent.if_nothing Enso_File.current_working_directory) . filter f-> f.asset_type == Enso_Asset_Type.Secret secrets_as_files.map f-> Enso_Secret.Value f.name f.id @@ -72,7 +73,7 @@ type Enso_Secret Arguments: - name: The name of the secret - parent: The parent folder for the secret. If `Nothing` then will check - in the root folder. + in the current working directory. get : Text -> Enso_File | Nothing -> Enso_Secret ! Not_Found get name:Text parent:(Enso_File | Nothing)=Nothing = Enso_Secret.list parent . find s-> s.name == name @@ -83,7 +84,7 @@ type Enso_Secret Arguments: - name: The name of the secret - parent: The parent folder for the secret. If `Nothing` then will check - in the root folder. + in the current working directory. exists : Text -> Enso_File | Nothing -> Boolean exists name:Text parent:(Enso_File | Nothing)=Nothing = Enso_Secret.list parent . any s-> s.name == name From 0451d4b87b14aa2a9d13f121ae990414cc7e5ad5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Fri, 19 Jan 2024 14:19:11 +0100 Subject: [PATCH 03/14] checkpoint 3 --- .../src/Network/Enso_Cloud/Enso_File_Spec.enso | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso b/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso index e8084a15a974..ed6a25eeac53 100644 --- a/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso +++ b/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso @@ -10,6 +10,17 @@ from enso_dev.Base_Tests.Network.Enso_Cloud.Cloud_Tests_Setup import with_retrie spec setup:Cloud_Tests_Setup = setup.with_prepared_environment <| Test.group "Enso Cloud Files" pending=setup.real_cloud_pending <| + Test.specify "should be able to list the root directory" <| + assets = Enso_File.root.list + # We don't a priori know the contents, so we can only check very generic properties + assets . should_be_a Vector + assets.each f-> f.should_be_a Enso_File + + ## We assume that it contains a test file `test_file.json` + TODO in future iterations this file will be created by the test suite itself, to make it self-contained + The file is expected to contain: + [1, 2, 3, "foo"] + assets.map .name . should_contain "test_file.json" Test.specify "should allow to create and delete a directory" <| my_name = "my_test_dir-" + (Random.uuid.take 5) my_dir = Enso_File.root.create_directory my_name From ca0c22cf8e7de9e26e37f5416e8a5969ae7c3a35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Fri, 19 Jan 2024 14:56:38 +0100 Subject: [PATCH 04/14] adding tests --- .../src/Data/Enso_Cloud/Enso_File.enso | 9 +++++ .../Network/Enso_Cloud/Enso_File_Spec.enso | 33 +++++++++++++++++-- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Enso_File.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Enso_File.enso index 26d26fd582df..14ab2d1f81b7 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Enso_File.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Enso_File.enso @@ -8,6 +8,7 @@ import project.Data.Text.Text import project.Data.Text.Text_Sub_Range.Text_Sub_Range import project.Data.Vector.Vector import project.Error.Error +import project.Errors.Common.Not_Found import project.Errors.File_Error.File_Error import project.Errors.Illegal_Argument.Illegal_Argument import project.Errors.Problem_Behavior.Problem_Behavior @@ -211,6 +212,14 @@ type Enso_File response = HTTP.post uri Request_Body.Empty HTTP_Method.Delete [auth_header] response.if_not_error <| Nothing + ## UNSTABLE + Resolves a file or directory within this directory. + / : Text -> Enso_File + / self (name : Text) -> Enso_File ! Not_Found = + if self.is_directory.not then Error.throw (Illegal_Argument.Error "/ can only be used for directories") else + if name.contains "/" then Error.throw (Illegal_Argument.Error "Resolving sub-paths (/) is not implemented. Temporary workaround: use the `/` operator multiple times.") else + self.list . find f-> f.name == name + ## PRIVATE list_assets parent = if parent.asset_type != Enso_Asset_Type.Directory then Error.throw (Illegal_Argument.Error "Only directories can be listed.") else auth_header = Utils.authorization_header diff --git a/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso b/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso index ed6a25eeac53..66e16e66fc0b 100644 --- a/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso +++ b/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso @@ -47,8 +47,37 @@ spec setup:Cloud_Tests_Setup = setup.with_prepared_environment <| # TODO the dir still shows as 'existing' after deletion, probably because it still is there in the Trash # my_dir.exists . should_be_false - Test.specify "should not allow to create a directory inside of a regular file" pending="TODO" <| - Error.throw "TODO" + Test.specify "should allow to find a file by name" <| + f = Enso_File.root / "test_file.json" + f.should_succeed + f.name . should_equal "test_file.json" + f.is_directory . should_be_false + f.exists . should_be_true + + Test.specify "should not find nonexistent files" <| + f = Enso_File.root / "nonexistent_file.json" + f.should_fail_with Not_Found + + Test.specify "should not allow to create a directory inside of a regular file" <| + test_file = Enso_File.root / "test_file.json" + test_file.exists . should_be_true + + r = test_file.create_directory "my_test_dir" + r.should_fail_with Illegal_Argument + + Test.specify "should delete all contents of a directory when deleting a directory" <| + dir1 = Enso_File.root.create_directory "my_test_dir1"+(Random.uuid.take 5) + dir1.should_succeed + + dir2 = dir1.create_directory "my_test_dir2" + dir2.should_succeed + + dir1.delete . should_succeed + + with_retries <| + dir1.exists . should_be_false + # The inner directory should also have been trashed if its parent is removed + dir2.exists . should_be_false Test.specify "should not allow to delete the root directory" <| Enso_File.root.delete . should_fail_with Illegal_Argument From 30d51114d46bb99a41ed46f340a7b9a028f7d0c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Fri, 19 Jan 2024 17:14:40 +0100 Subject: [PATCH 05/14] tests for file reading --- .../Network/Enso_Cloud/Enso_File_Spec.enso | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso b/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso index 66e16e66fc0b..3921342016c5 100644 --- a/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso +++ b/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso @@ -48,6 +48,7 @@ spec setup:Cloud_Tests_Setup = setup.with_prepared_environment <| # my_dir.exists . should_be_false Test.specify "should allow to find a file by name" <| + # TODO the file should be created programmatically when write is implemented f = Enso_File.root / "test_file.json" f.should_succeed f.name . should_equal "test_file.json" @@ -59,6 +60,7 @@ spec setup:Cloud_Tests_Setup = setup.with_prepared_environment <| f.should_fail_with Not_Found Test.specify "should not allow to create a directory inside of a regular file" <| + # TODO the file should be created programmatically when write is implemented test_file = Enso_File.root / "test_file.json" test_file.exists . should_be_true @@ -82,4 +84,32 @@ spec setup:Cloud_Tests_Setup = setup.with_prepared_environment <| Test.specify "should not allow to delete the root directory" <| Enso_File.root.delete . should_fail_with Illegal_Argument + Test.specify "should be able to create and delete a file" pending="TODO: Cloud file write support" <| + Error.throw "TODO" + + expected_file_text = '[1, 2, 3, "foo"]' + Test.specify "should be able to read and decode a file using various formats" <| + # TODO the file should be created programmatically when write is implemented + test_file = Enso_File.root / "test_file.json" + test_file.exists . should_be_true + + test_file.read Plain_Text . should_equal expected_file_text + + # auto-detection of JSON format: + json = test_file.read + json.should_be_a Vector + json.should_equal [1, 2, 3, "foo"] + + test_file.read_bytes . should_equal expected_file_text.utf_8 + + Test.specify "should be able to open a file as input stream" <| + test_file = Enso_File.root / "test_file.json" + test_file.exists . should_be_true + + bytes = test_file.with_input_stream [File_Access.Read] stream-> + stream.read_all_bytes + + bytes.should_equal expected_file_text.utf_8 + + main = Test_Suite.run_main (spec Cloud_Tests_Setup.prepare) From 4729a8bc5bb424f24c9ab4395d53316d5f758581 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Fri, 19 Jan 2024 17:46:47 +0100 Subject: [PATCH 06/14] more tests --- .../Base/0.0.0-dev/src/System/File.enso | 4 +- .../Network/Enso_Cloud/Enso_File_Spec.enso | 59 +++++++++++++++++++ 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/System/File.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/System/File.enso index 0138d6142f9d..61cf37fdf1be 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/System/File.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/System/File.enso @@ -428,7 +428,7 @@ type File import Standard.Examples example_exists = Examples.csv.creation_time - creation_time : Time_Of_Day ! File_Error + creation_time : Date_Time ! File_Error creation_time self = File_Error.handle_java_exceptions self <| self.creation_time_builtin @@ -441,7 +441,7 @@ type File import Standard.Examples example_exists = Examples.csv.last_modified_time - last_modified_time : Time_Of_Day ! File_Error + last_modified_time : Date_Time ! File_Error last_modified_time self = File_Error.handle_java_exceptions self <| self.last_modified_time_builtin diff --git a/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso b/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso index 3921342016c5..44abc5e2d05c 100644 --- a/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso +++ b/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso @@ -111,5 +111,64 @@ spec setup:Cloud_Tests_Setup = setup.with_prepared_environment <| bytes.should_equal expected_file_text.utf_8 + Test.specify "should be able to read file metadata" <| + Enso_File.root.exists . should_be_true + # TODO what semantics we choose here? + Enso_File.root.parent . should_equal Enso_File.root + Enso_File.root.path . should_equal "enso://" + Enso_File.root.name . should_equal "/" + + # TODO this structure should be created once we get file writing + dir = Enso_File.root / "test-directory" + dir.exists.should_be_true + + dir.name . should_equal "test-directory" + dir.extension . should_fail_with Illegal_Argument + dir.size . should_fail_with Illegal_Argument + dir.path . should_equal "enso://test-directory/" + dir.parent . should_equal Enso_File.root + dir.is_child_of Enso_File.root . should_be_true + Enso_File.root.is_child_of dir . should_be_false + dir.is_directory.should_be_true + dir.is_regular_file.should_be_false + dir.creation_time . should_be_a Date_Time + dir.last_modified_time . should_be_a Date_Time + + nested_file = dir / "another.txt" + nested_file.exists.should_be_true + + nested_file.name . should_equal "another.txt" + nested_file.extension . should_equal "txt" + nested_file.size . should_equal nested_file.read_bytes.length + nested_file.path . should_equal "enso://test-directory/another.txt" + nested_file.parent . should_equal dir + nested_file.is_child_of dir . should_be_true + nested_file.is_child_of Enso_File.root . should_be_true + nested_file.is_directory.should_be_false + nested_file.is_regular_file.should_be_true + nested_file.creation_time . should_be_a Date_Time + nested_file.last_modified_time . should_be_a Date_Time + + Test.specify "should be able to read other file metadata" pending="TODO do we want that?" <| + nested_file = Enso_File.root / "test-directory" / "another.txt" + + nested_file.is_absolute.should_be_true + nested_file.absolute . should_equal nested_file + nested_file.normalize . should_equal nested_file + nested_file.posix_permissions . should_be_a File_Permissions + nested_file.is_writable . should_be_a Boolean + + Test.specify "should be able to copy a file" pending="TODO Cloud file writing" <| + nested_file = Enso_File.root / "test-directory" / "another.txt" + ## TODO currently `/` only works if the file already exists, so how will we construct a path for a copy target? + We either need to make all files just paths (like on regular File), or we need to have concrete files (with ID) and abstract files (just path). + new_file = nested_file.copy_to "TODO this needs design" + nested_file.exists.should_be_true + new_file.exists.should_be_true + + Test.specify "should be able to move a file" pending="TODO Cloud file writing" <| + nested_file = Enso_File.root / "test-directory" / "another.txt" + nested_file.move_to "TODO this needs design" + nested_file.exists . should_be_false main = Test_Suite.run_main (spec Cloud_Tests_Setup.prepare) From f8596ea788c4d32c47c35cf35b32271e7e0f4467 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Fri, 19 Jan 2024 17:58:17 +0100 Subject: [PATCH 07/14] more tests 2 --- .../src/Data/Enso_Cloud/Enso_File.enso | 7 ++++- .../Network/Enso_Cloud/Enso_File_Spec.enso | 28 +++++++++++-------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Enso_File.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Enso_File.enso index 14ab2d1f81b7..355892bfc236 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Enso_File.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Enso_File.enso @@ -32,7 +32,7 @@ type Enso_File ## Represents the root folder of the current users. root : Enso_File - root = Enso_File.Value "" "" "" Enso_Asset_Type.Directory + root = Enso_File.Value "/" "" "" Enso_Asset_Type.Directory ## Represents the current working directory. @@ -72,6 +72,11 @@ type Enso_File is_directory : Boolean is_directory self = self.asset_type == Enso_Asset_Type.Directory + ## GROUP Metadata + Checks if this is a regular file + is_regular_file : Boolean + is_regular_file self = self.asset_type == Enso_Asset_Type.File + ## PRIVATE ADVANCED Creates a new output stream for this file and runs the specified action diff --git a/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso b/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso index 44abc5e2d05c..34327e6b3006 100644 --- a/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso +++ b/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso @@ -113,39 +113,43 @@ spec setup:Cloud_Tests_Setup = setup.with_prepared_environment <| Test.specify "should be able to read file metadata" <| Enso_File.root.exists . should_be_true - # TODO what semantics we choose here? - Enso_File.root.parent . should_equal Enso_File.root - Enso_File.root.path . should_equal "enso://" Enso_File.root.name . should_equal "/" # TODO this structure should be created once we get file writing dir = Enso_File.root / "test-directory" dir.exists.should_be_true - dir.name . should_equal "test-directory" dir.extension . should_fail_with Illegal_Argument + dir.is_directory.should_be_true + dir.is_regular_file.should_be_false + + nested_file = dir / "another.txt" + nested_file.exists.should_be_true + nested_file.name . should_equal "another.txt" + nested_file.extension . should_equal "txt" + nested_file.is_directory.should_be_false + nested_file.is_regular_file.should_be_true + + Test.specify "should be able to read file metadata (v2)" pending="TODO Enso_File more detailed metadata" <| + # TODO what semantics we choose here? + Enso_File.root.parent . should_equal Enso_File.root + Enso_File.root.path . should_equal "enso://" + + dir = Enso_File.root / "test-directory" dir.size . should_fail_with Illegal_Argument dir.path . should_equal "enso://test-directory/" dir.parent . should_equal Enso_File.root dir.is_child_of Enso_File.root . should_be_true Enso_File.root.is_child_of dir . should_be_false - dir.is_directory.should_be_true - dir.is_regular_file.should_be_false dir.creation_time . should_be_a Date_Time dir.last_modified_time . should_be_a Date_Time nested_file = dir / "another.txt" - nested_file.exists.should_be_true - - nested_file.name . should_equal "another.txt" - nested_file.extension . should_equal "txt" nested_file.size . should_equal nested_file.read_bytes.length nested_file.path . should_equal "enso://test-directory/another.txt" nested_file.parent . should_equal dir nested_file.is_child_of dir . should_be_true nested_file.is_child_of Enso_File.root . should_be_true - nested_file.is_directory.should_be_false - nested_file.is_regular_file.should_be_true nested_file.creation_time . should_be_a Date_Time nested_file.last_modified_time . should_be_a Date_Time From 8365bc6d38cb78cf726d45c79d3595f4cb429452 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Fri, 19 Jan 2024 18:04:57 +0100 Subject: [PATCH 08/14] fixes --- distribution/lib/Standard/Base/0.0.0-dev/src/System/File.enso | 2 +- test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/System/File.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/System/File.enso index 61cf37fdf1be..60640a271a8c 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/System/File.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/System/File.enso @@ -8,7 +8,7 @@ import project.Data.Text.Extensions import project.Data.Text.Matching_Mode.Matching_Mode import project.Data.Text.Text import project.Data.Text.Text_Sub_Range.Text_Sub_Range -import project.Data.Time.Time_Of_Day.Time_Of_Day +import project.Data.Time.Date_Time.Date_Time import project.Data.Vector.Vector import project.Error.Error import project.Errors.Common.Dry_Run_Operation diff --git a/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso b/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso index 34327e6b3006..ee04803266a7 100644 --- a/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso +++ b/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso @@ -1,4 +1,5 @@ from Standard.Base import all +import Standard.Base.Errors.Common.Not_Found import Standard.Base.Errors.Illegal_Argument.Illegal_Argument from Standard.Test import Test, Test_Suite From fe2d3b3bcabf316a9adbce4891be101cfbf33250 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Fri, 19 Jan 2024 18:39:52 +0100 Subject: [PATCH 09/14] updating to API changes --- .../Base/0.0.0-dev/src/Data/Enso_Cloud/Enso_File.enso | 11 +++++++---- .../src/Network/Enso_Cloud/Enso_File_Spec.enso | 4 ++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Enso_File.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Enso_File.enso index 355892bfc236..938716b13616 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Enso_File.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Enso_File.enso @@ -11,6 +11,7 @@ import project.Error.Error import project.Errors.Common.Not_Found import project.Errors.File_Error.File_Error import project.Errors.Illegal_Argument.Illegal_Argument +import project.Errors.Illegal_State.Illegal_State import project.Errors.Problem_Behavior.Problem_Behavior import project.Errors.Unimplemented.Unimplemented import project.Network.HTTP.HTTP @@ -115,11 +116,11 @@ type Enso_File response = HTTP.fetch self.internal_uri HTTP_Method.Get [auth_header] response.if_not_error <| js_object = response.decode_as_json - path = js_object.get "path" - if path.is_nothing then Error.throw (Illegal_Argument.Error "Invalid JSON for an Enso_File.") else + path = js_object.get "file" . get "path" + if path.is_nothing then Error.throw (Illegal_State.Error "Invalid JSON for an Enso_File: "+js_object.to_text) else url = path.replace "s3://production-enso-organizations-files/" "https://production-enso-organizations-files.s3.eu-west-1.amazonaws.com/" response = HTTP.fetch url HTTP_Method.Get [] - response.if_not_error <| response.with_stream action + response.if_not_error <| response.body.with_stream action ## ALIAS load, open GROUP Input @@ -149,7 +150,9 @@ type Enso_File if real_format == Nothing then Error.throw (File_Error.Unsupported_Type self) else self.read real_format on_problems _ -> - metadata = File_Format_Metadata.Value file_name=self.name + # TODO this is just a placeholder, until we implement the proper path + path = "enso://"+self.id + metadata = File_Format_Metadata.Value path=path name=self.name self.with_input_stream [File_Access.Read] (stream-> format.read_stream stream metadata) ## ALIAS load bytes, open bytes diff --git a/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso b/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso index ee04803266a7..93537b9a9df3 100644 --- a/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso +++ b/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso @@ -68,7 +68,7 @@ spec setup:Cloud_Tests_Setup = setup.with_prepared_environment <| r = test_file.create_directory "my_test_dir" r.should_fail_with Illegal_Argument - Test.specify "should delete all contents of a directory when deleting a directory" <| + Test.specify "should delete all contents of a directory when deleting a directory" pending="TODO discuss recursive delete" <| dir1 = Enso_File.root.create_directory "my_test_dir1"+(Random.uuid.take 5) dir1.should_succeed @@ -127,7 +127,7 @@ spec setup:Cloud_Tests_Setup = setup.with_prepared_environment <| nested_file = dir / "another.txt" nested_file.exists.should_be_true nested_file.name . should_equal "another.txt" - nested_file.extension . should_equal "txt" + nested_file.extension . should_equal ".txt" nested_file.is_directory.should_be_false nested_file.is_regular_file.should_be_true From 08ad26e2e0b91f1315c0368c2abded89c7b2e597 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Fri, 19 Jan 2024 19:07:01 +0100 Subject: [PATCH 10/14] implement Enso_File.size --- .../src/Data/Enso_Cloud/Enso_File.enso | 36 ++++++++++++++----- .../Network/Enso_Cloud/Enso_File_Spec.enso | 4 +-- 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Enso_File.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Enso_File.enso index 938716b13616..d2a04d961e3c 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Enso_File.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Enso_File.enso @@ -2,6 +2,7 @@ import project.Any.Any import project.Data.Enso_Cloud.Utils import project.Data.Index_Sub_Range.Index_Sub_Range import project.Data.Json.JS_Object +import project.Data.Numbers.Integer import project.Data.Text.Encoding.Encoding import project.Data.Text.Matching_Mode.Matching_Mode import project.Data.Text.Text @@ -68,6 +69,15 @@ type Enso_File response = HTTP.fetch self.internal_uri HTTP_Method.Get [auth_header] response.code.is_success + ## GROUP Metadata + Gets the size of a file in bytes. + size : Integer + size self = if self.is_regular_file.not then Error.throw (Illegal_Argument.Error "`size` can only be queried for regular files.") else + s3_url = get_s3_url_for_file self + response = HTTP.fetch s3_url HTTP_Method.Head [] + content_length = response.get_header "content-length" if_missing=(Error.throw (Illegal_State.Error "Cannot get file size: missing `Content-Lentth` header.")) + Integer.parse content_length + ## GROUP Metadata Checks if this is a folder is_directory : Boolean @@ -112,15 +122,9 @@ type Enso_File with_input_stream : Vector File_Access -> (Input_Stream -> Any ! File_Error) -> Any ! File_Error | Illegal_Argument with_input_stream self open_options action = if self.asset_type != Enso_Asset_Type.File then Error.throw (Illegal_Argument.Error "Only files can be opened as a stream.") else if (open_options != [File_Access.Read]) then Error.throw (Illegal_Argument.Error "Files can only be opened for reading.") else - auth_header = Utils.authorization_header - response = HTTP.fetch self.internal_uri HTTP_Method.Get [auth_header] - response.if_not_error <| - js_object = response.decode_as_json - path = js_object.get "file" . get "path" - if path.is_nothing then Error.throw (Illegal_State.Error "Invalid JSON for an Enso_File: "+js_object.to_text) else - url = path.replace "s3://production-enso-organizations-files/" "https://production-enso-organizations-files.s3.eu-west-1.amazonaws.com/" - response = HTTP.fetch url HTTP_Method.Get [] - response.if_not_error <| response.body.with_stream action + s3_url = get_s3_url_for_file self + response = HTTP.fetch s3_url HTTP_Method.Get [] + response.if_not_error <| response.body.with_stream action ## ALIAS load, open GROUP Input @@ -272,3 +276,17 @@ Enso_Asset_Type.from (that:Text) = case that of ## PRIVATE File_Format_Metadata.from (that:Enso_File) = File_Format_Metadata.Value Nothing that.name (that.extension.catch _->Nothing) + +## PRIVATE +get_file_description file:Enso_File -> JS_Object = + auth_header = Utils.authorization_header + response = HTTP.fetch file.internal_uri HTTP_Method.Get [auth_header] + response.if_not_error <| + js_object = response.decode_as_json + js_object.get "file" if_missing=(Error.throw (Illegal_State.Error "Invalid JSON for an Enso_File (missing `file` field): "+js_object.to_text)) + +## PRIVATE +get_s3_url_for_file file:Enso_File -> Text = + file_description = get_file_description file + path = file_description.get "path" if_missing=(Error.throw (Illegal_State.Error "Invalid JSON for an Enso_File (missing `path`): "+file_description.to_text)) + path.replace "s3://production-enso-organizations-files/" "https://production-enso-organizations-files.s3.eu-west-1.amazonaws.com/" diff --git a/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso b/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso index 93537b9a9df3..014874da07a3 100644 --- a/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso +++ b/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso @@ -123,6 +123,7 @@ spec setup:Cloud_Tests_Setup = setup.with_prepared_environment <| dir.extension . should_fail_with Illegal_Argument dir.is_directory.should_be_true dir.is_regular_file.should_be_false + dir.size . should_fail_with Illegal_Argument nested_file = dir / "another.txt" nested_file.exists.should_be_true @@ -130,6 +131,7 @@ spec setup:Cloud_Tests_Setup = setup.with_prepared_environment <| nested_file.extension . should_equal ".txt" nested_file.is_directory.should_be_false nested_file.is_regular_file.should_be_true + nested_file.size . should_equal nested_file.read_bytes.length Test.specify "should be able to read file metadata (v2)" pending="TODO Enso_File more detailed metadata" <| # TODO what semantics we choose here? @@ -137,7 +139,6 @@ spec setup:Cloud_Tests_Setup = setup.with_prepared_environment <| Enso_File.root.path . should_equal "enso://" dir = Enso_File.root / "test-directory" - dir.size . should_fail_with Illegal_Argument dir.path . should_equal "enso://test-directory/" dir.parent . should_equal Enso_File.root dir.is_child_of Enso_File.root . should_be_true @@ -146,7 +147,6 @@ spec setup:Cloud_Tests_Setup = setup.with_prepared_environment <| dir.last_modified_time . should_be_a Date_Time nested_file = dir / "another.txt" - nested_file.size . should_equal nested_file.read_bytes.length nested_file.path . should_equal "enso://test-directory/another.txt" nested_file.parent . should_equal dir nested_file.is_child_of dir . should_be_true From 382ae709effd575c8b6e8e8b17c2721abc7554a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Fri, 19 Jan 2024 19:15:55 +0100 Subject: [PATCH 11/14] add method stubs for some other methods --- .../src/Data/Enso_Cloud/Enso_File.enso | 28 +++++++++++++++++++ .../Network/Enso_Cloud/Enso_File_Spec.enso | 3 +- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Enso_File.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Enso_File.enso index d2a04d961e3c..b012b69d29de 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Enso_File.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Enso_File.enso @@ -7,6 +7,7 @@ import project.Data.Text.Encoding.Encoding import project.Data.Text.Matching_Mode.Matching_Mode import project.Data.Text.Text import project.Data.Text.Text_Sub_Range.Text_Sub_Range +import project.Data.Time.Date_Time.Date_Time import project.Data.Vector.Vector import project.Error.Error import project.Errors.Common.Not_Found @@ -78,6 +79,16 @@ type Enso_File content_length = response.get_header "content-length" if_missing=(Error.throw (Illegal_State.Error "Cannot get file size: missing `Content-Lentth` header.")) Integer.parse content_length + ## GROUP Metadata + Gets the creation time of a file. + creation_time : Date_Time + creation_time self = Unimplemented.throw "Enso_File.creation_time is not implemented" + + ## GROUP Metadata + Gets the last modified time of a file. + last_modified_time : Date_Time + last_modified_time self = Unimplemented.throw "Enso_File.creation_time is not implemented" + ## GROUP Metadata Checks if this is a folder is_directory : Boolean @@ -88,6 +99,23 @@ type Enso_File is_regular_file : Boolean is_regular_file self = self.asset_type == Enso_Asset_Type.File + ## GROUP Metadata + Finds the parent Enso_File for this file. + parent : Enso_File | Nothing + parent self = Unimplemented.throw "Enso_File.parent is not implemented" + + ## GROUP Metadata + Returns the path of this file. + path : Text + path self = Unimplemented.throw "Enso_File.path is not implemented" + + ## GROUP Metadata + Checks if `self` is a child path of `other`. + is_child_of : Enso_File -> Boolean + is_child_of self (other : Enso_File) = + _ = other + Unimplemented.throw "Enso_File.is_child_of is not implemented" + ## PRIVATE ADVANCED Creates a new output stream for this file and runs the specified action diff --git a/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso b/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso index 014874da07a3..d9c8178cf805 100644 --- a/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso +++ b/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso @@ -134,8 +134,7 @@ spec setup:Cloud_Tests_Setup = setup.with_prepared_environment <| nested_file.size . should_equal nested_file.read_bytes.length Test.specify "should be able to read file metadata (v2)" pending="TODO Enso_File more detailed metadata" <| - # TODO what semantics we choose here? - Enso_File.root.parent . should_equal Enso_File.root + Enso_File.root.parent . should_equal Nothing Enso_File.root.path . should_equal "enso://" dir = Enso_File.root / "test-directory" From 73bc50133b10da52954d955ac07e75311440dcdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Fri, 26 Jan 2024 13:24:17 +0100 Subject: [PATCH 12/14] add current working dir --- .../src/Data/Enso_Cloud/Enso_File.enso | 7 +++--- .../0.0.0-dev/src/Data/Enso_Cloud/Utils.enso | 5 ++++ .../enso_cloud/AuthenticationProvider.java | 23 +++++++++++++++++++ 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Enso_File.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Enso_File.enso index b012b69d29de..535a00851f6d 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Enso_File.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Enso_File.enso @@ -31,7 +31,7 @@ from project.System.File_Format import Auto_Detect, Bytes, File_Format, Plain_Te type Enso_File ## PRIVATE Represents a file or folder within the Enso cloud. - Value name:Text id:Text organisation:Text asset_type:Enso_Asset_Type + Value name:Text id:Text organization:Text asset_type:Enso_Asset_Type ## Represents the root folder of the current users. root : Enso_File @@ -46,8 +46,9 @@ type Enso_File directory. current_working_directory : Enso_File current_working_directory = - # TODO - Enso_File.root + java_dir = Utils.internal_cloud_project_directory + if java_dir.is_nothing then Enso_File.root else + Enso_File.Value java_dir.name java_dir.id java_dir.organizationId Enso_Asset_Type.Directory ## PRIVATE Target URI for the api diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Utils.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Utils.enso index 500271840223..dc5e4da916f5 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Utils.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Utils.enso @@ -48,6 +48,11 @@ projects_api = cloud_root_uri + "projects" Root address for Secrets API secrets_api = cloud_root_uri + "secrets" +## PRIVATE + The current project directory that will be used as the working directory, + if the user is running in the Cloud. +internal_cloud_project_directory = AuthenticationProvider.getCurrentWorkingDirectory + ## PRIVATE flush_caches : Nothing flush_caches = AuthenticationProvider.flushCloudCaches diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/AuthenticationProvider.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/AuthenticationProvider.java index 57f790b2a455..e769609d91ad 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/AuthenticationProvider.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/AuthenticationProvider.java @@ -22,7 +22,30 @@ public static String getAPIRootURI() { return uriWithSlash; } + public record CloudWorkingDirectory(String name, String id, String organizationId) {} + + public static CloudWorkingDirectory getCurrentWorkingDirectory() { + if (cachedWorkingDirectory != null) { + return cachedWorkingDirectory; + } + + String directoryId = Environment_Utils.get_environment_variable("ENSO_PROJECT_PATH"); + if (directoryId == null) { + // No current working directory is set + return null; + } + + // TODO we should be able to fetch the name and organizationId from the cloud: + String directoryName = "???"; + String organizationId = ""; + cachedWorkingDirectory = new CloudWorkingDirectory(directoryName, directoryId, organizationId); + return cachedWorkingDirectory; + } + + private static CloudWorkingDirectory cachedWorkingDirectory = null; + public static void flushCloudCaches() { EnsoSecretReader.flushCache(); + cachedWorkingDirectory = null; } } From 98a080fef7a700832ca0aae92be364897dfcc2c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Fri, 26 Jan 2024 13:53:47 +0100 Subject: [PATCH 13/14] add test --- .../Network/Enso_Cloud/Enso_File_Spec.enso | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso b/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso index d9c8178cf805..d484e622bb4a 100644 --- a/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso +++ b/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso @@ -22,6 +22,7 @@ spec setup:Cloud_Tests_Setup = setup.with_prepared_environment <| The file is expected to contain: [1, 2, 3, "foo"] assets.map .name . should_contain "test_file.json" + Test.specify "should allow to create and delete a directory" <| my_name = "my_test_dir-" + (Random.uuid.take 5) my_dir = Enso_File.root.create_directory my_name @@ -48,6 +49,27 @@ spec setup:Cloud_Tests_Setup = setup.with_prepared_environment <| # TODO the dir still shows as 'existing' after deletion, probably because it still is there in the Trash # my_dir.exists . should_be_false + Test.specify "should set the current working directory by environment variable" <| + # If nothing set, defaults to root: + Enso_File.current_working_directory . should_equal Enso_File.root + + subdir = Enso_File.root.create_directory "my_test_CWD-" + (Random.uuid.take 5) + cleanup = + Enso_User.flush_caches + subdir.delete + Panic.with_finalizer cleanup <| + Test_Environment.unsafe_with_environment_override "ENSO_PROJECT_PATH" subdir.id <| + # Flush caches to ensure fresh dir is used + Enso_User.flush_caches + + # TODO this should work, but further API changes are needed + # Enso_File.current_working_directory . should_equal subdir + # So now just checking id: + Enso_File.current_working_directory.id . should_equal subdir.id + + # It should be back to default afterwards: + Enso_File.current_working_directory . should_equal Enso_File.root + Test.specify "should allow to find a file by name" <| # TODO the file should be created programmatically when write is implemented f = Enso_File.root / "test_file.json" From 55a878024ad3e5c4fd19c0efd061049c86441301 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Fri, 26 Jan 2024 14:21:21 +0100 Subject: [PATCH 14/14] fixes --- test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso b/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso index d484e622bb4a..446503ee6b68 100644 --- a/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso +++ b/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso @@ -53,7 +53,7 @@ spec setup:Cloud_Tests_Setup = setup.with_prepared_environment <| # If nothing set, defaults to root: Enso_File.current_working_directory . should_equal Enso_File.root - subdir = Enso_File.root.create_directory "my_test_CWD-" + (Random.uuid.take 5) + subdir = Enso_File.root.create_directory "my_test_CWD-"+(Random.uuid.take 5) cleanup = Enso_User.flush_caches subdir.delete @@ -155,7 +155,7 @@ spec setup:Cloud_Tests_Setup = setup.with_prepared_environment <| nested_file.is_regular_file.should_be_true nested_file.size . should_equal nested_file.read_bytes.length - Test.specify "should be able to read file metadata (v2)" pending="TODO Enso_File more detailed metadata" <| + Test.specify "should be able to read file metadata (v2)" pending="TODO Enso_File more detailed metadata, waiting on https://github.com/enso-org/cloud-v2/issues/870" <| Enso_File.root.parent . should_equal Nothing Enso_File.root.path . should_equal "enso://" @@ -175,7 +175,7 @@ spec setup:Cloud_Tests_Setup = setup.with_prepared_environment <| nested_file.creation_time . should_be_a Date_Time nested_file.last_modified_time . should_be_a Date_Time - Test.specify "should be able to read other file metadata" pending="TODO do we want that?" <| + Test.specify "should be able to read other file metadata" pending="TODO needs further design" <| nested_file = Enso_File.root / "test-directory" / "another.txt" nested_file.is_absolute.should_be_true