Audit Logs for Postgres connections opened through a data link (#9873)

- Closes #9599
- Implemented API for sending audit logs to the cloud on a background thread.
- If the Postgres connection is opened through a datalink, its internal JDBC connection is replaced by a wrapper that reports executed queries to the audit log.
- Also introduces `EnsoMeta` - a helper Java class that can be used in our helper libraries to access Enso types.
- I have replaced the common pattern scattered throughout the codebase with calls to this 'library' to avoid repetitive code.
- Refactored `Table.display` to share code between in-memory and DB - it was needed as the function stopped working for `DB_Table` after adding making the `Table` constructor `private`.
- Clearer error when reading a SQLite database from a remote file (tells the user to download it first).
- Follow up - correlate asset id of the data link:
#9869
- Follow up - include project name (once bug is fixed):
#9875
- Some problems/improvements of the audit log:
- The audit log system is not yet ready for high throughput of logs
#9870
- The logs may be lost if `System.exit` is used
#9871

Author: radeusgd

CHANGELOG.md

@@ -661,6 +661,8 @@
 - [Added `Vector.build_multiple`, and better for support for errors and warnings
   inside `Vector.build` and `Vector.build_multiple`.][9766]
 - [Added `Vector.duplicates`.][9917]
+- [Log operations performed on a Postgres database connection obtained through a
+  Data Link.][9873]
 
 [debug-shortcuts]:
   https://github.com/enso-org/enso/blob/develop/app/gui/docs/product/shortcuts.md#debug
@@ -970,6 +972,7 @@
 [9750]: https://github.com/enso-org/enso/pull/9750
 [9766]: https://github.com/enso-org/enso/pull/9766
 [9917]: https://github.com/enso-org/enso/pull/9917
+[9873]: https://github.com/enso-org/enso/pull/9873
 
 #### Enso Compiler
 

distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Enso_User.enso

@@ -4,12 +4,15 @@ import project.Data.Time.Duration.Duration
 import project.Data.Vector.Vector
 import project.Enso_Cloud.Enso_File.Enso_Asset_Type
 import project.Enso_Cloud.Enso_File.Enso_File
+import project.Enso_Cloud.Errors.Not_Logged_In
+import project.Enso_Cloud.Internal.Authentication
 import project.Enso_Cloud.Internal.Utils
 import project.Error.Error
 import project.Errors.Illegal_Argument.Illegal_Argument
 import project.Network.HTTP.HTTP
 import project.Network.HTTP.HTTP_Method.HTTP_Method
 import project.Nothing.Nothing
+import project.Panic.Panic
 from project.Data.Boolean import Boolean, False, True
 from project.Enso_Cloud.Public_Utils import get_optional_field, get_required_field
 
@@ -29,12 +32,17 @@ type Enso_User
 
     ## ICON people
        Fetch the current user.
-    current : Enso_User
-    current =
+    current -> Enso_User =
         Utils.get_cached "users/me" cache_duration=(Duration.new minutes=120) <|
             json = Utils.http_request_as_json HTTP_Method.Get (Utils.cloud_root_uri + "users/me")
             Enso_User.from json
 
+    ## PRIVATE
+       Checks if the user is logged in.
+    is_logged_in -> Boolean =
+        Panic.catch Not_Logged_In handler=(_->False) <|
+            Authentication.get_access_token.is_error.not
+
     ## ICON people
        Lists all known users.
     list : Vector Enso_User

distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Internal/Audit_Log.enso

@@ -0,0 +1,37 @@
+import project.Data.Json.JS_Object
+import project.Data.Text.Text
+import project.Errors.Illegal_Argument.Illegal_Argument
+import project.Nothing.Nothing
+from project.Data.Boolean import Boolean, False, True
+
+polyglot java import org.enso.base.enso_cloud.audit.AuditLog
+
+## PRIVATE
+type Audit_Log
+    ## PRIVATE
+       Reports an event to the audit log.
+       The event is submitted asynchronously.
+
+       Arguments:
+       - event_type: The type of the event.
+       - message: The message associated with the event.
+       - metadata: Additional metadata to include with the event.
+         Note that it should be a JS object and it should _not_ contain fields
+         that are restricted. These fields are added to the metadata
+         automatically.
+       - async: Whether to submit the event asynchronously.
+         Defaults to True.
+
+       ? Restricted Fields
+
+         The following fields are added by the system and should not be included
+         in the provided metadata:
+         - `type`
+         - `operation`
+         - `localTimestamp`
+         - `projectName`
+    report_event event_type:Text message:Text (metadata:JS_Object = JS_Object.from_pairs []) (async : Boolean = True) -> Nothing =
+        Illegal_Argument.handle_java_exception <|
+            case async of
+                True -> AuditLog.logAsync event_type message metadata.object_node
+                False -> AuditLog.logSynchronously event_type message metadata.object_node

distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Public_Utils.enso

@@ -2,6 +2,7 @@ import project.Any.Any
 import project.Data.Json.JS_Object
 import project.Data.Text.Text
 import project.Enso_Cloud.Errors.Enso_Cloud_Error
+import project.Enso_Cloud.Internal.Utils
 import project.Error.Error
 import project.Meta
 import project.Nothing.Nothing
@@ -57,3 +58,11 @@ get_optional_field (key : Text) js_object (~if_missing = Nothing) (show_value :
     _ ->
         representation = if show_value then js_object.to_display_text else Meta.type_of js_object . to_display_text
         Error.throw (Enso_Cloud_Error.Invalid_Response_Payload "Expected a JSON object, but got "+representation+".")
+
+## PRIVATE
+   UNSTABLE
+   Re-exports parts of the functionality of `http_request_as_json` function that
+   is needed in tests.
+   It should not be used anywhere else and may be removed in the near future.
+cloud_http_request_for_test method url_suffix =
+    Utils.http_request_as_json method Utils.cloud_root_uri+url_suffix

distribution/lib/Standard/Base/0.0.0-dev/src/System/File_Format.enso

@@ -96,6 +96,13 @@ type File_Format
         _ = [file, on_problems]
         Unimplemented.throw "This is an interface only."
 
+    ## PRIVATE
+       Implements decoding the format from a stream.
+    read_stream : Input_Stream -> File_Format_Metadata -> Any
+    read_stream self stream:Input_Stream (metadata : File_Format_Metadata) =
+        _ = [stream, metadata]
+        Unimplemented.throw "This is an interface only."
+
     ## PRIVATE
     default_widget : Widget
     default_widget =

distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Data_Link/Postgres_Data_Link.enso

@@ -45,7 +45,9 @@ type Postgres_Data_Link
     read self (format = Auto_Detect) (on_problems : Problem_Behavior) =
         _ = on_problems
         if format != Auto_Detect then Error.throw (Illegal_Argument.Error "Only the default Auto_Detect format should be used with a Postgres Data Link, because it does not point to a file resource, but a database entity, so setting a file format for it is meaningless.") else
-            default_options = Connection_Options.Value
+            # TODO add related asset id here: https://github.com/enso-org/enso/issues/9869
+            audit_mode = if Enso_User.is_logged_in then "cloud" else "local"
+            default_options = Connection_Options.Value [["enso.internal.audit", audit_mode]]
             connection = self.details.connect default_options
             case self of
                 Postgres_Data_Link.Connection _ -> connection

distribution/lib/Standard/Database/0.0.0-dev/src/Connection/SQLite_Format.enso

@@ -2,6 +2,7 @@ from Standard.Base import all
 import Standard.Base.Errors.Illegal_Argument.Illegal_Argument
 import Standard.Base.System.File.Generic.Writable_File.Writable_File
 import Standard.Base.System.File_Format_Metadata.File_Format_Metadata
+import Standard.Base.System.Input_Stream.Input_Stream
 from Standard.Base.Metadata.Choice import Option
 
 import project.Connection.Database
@@ -48,6 +49,12 @@ type SQLite_Format
         _ = [on_problems]
         Database.connect (SQLite.From_File file)
 
+    ## PRIVATE
+    read_stream : Input_Stream -> File_Format_Metadata -> Any
+    read_stream self stream metadata =
+        _ = [stream, metadata]
+        Error.throw (Illegal_Argument.Error "Cannot connect to a SQLite database backed by a stream. Save it to a local file first.")
+
 ## PRIVATE
    Based on the File Format definition at: https://www.sqlite.org/fileformat.html
 magic_header_string =

distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso

@@ -23,6 +23,7 @@ import Standard.Table.Internal.Add_Row_Number
 import Standard.Table.Internal.Aggregate_Column_Helper
 import Standard.Table.Internal.Column_Naming_Helper.Column_Naming_Helper
 import Standard.Table.Internal.Constant_Column.Constant_Column
+import Standard.Table.Internal.Display_Helpers
 import Standard.Table.Internal.Join_Kind_Cross.Join_Kind_Cross
 import Standard.Table.Internal.Problem_Builder.Problem_Builder
 import Standard.Table.Internal.Replace_Helpers
@@ -35,10 +36,8 @@ import Standard.Table.Internal.Widget_Helpers
 import Standard.Table.Match_Columns as Match_Columns_Helpers
 import Standard.Table.Row.Row
 from Standard.Table import Aggregate_Column, Auto, Blank_Selector, Column_Ref, Data_Formatter, Join_Condition, Join_Kind, Match_Columns, Position, Previous_Value, Report_Unmatched, Set_Mode, Simple_Expression, Sort_Column, Table, Value_Type
-from Standard.Table.Column import get_item_string, normalize_string_for_display
 from Standard.Table.Errors import all
 from Standard.Table.Internal.Filter_Condition_Helpers import make_filter_column
-from Standard.Table.Table import print_table
 
 import project.Connection.Connection.Connection
 import project.DB_Column.DB_Column
@@ -97,9 +96,12 @@ type DB_Table
          - format_terminal: whether ANSI-terminal formatting should be used
     display : Integer -> Boolean -> Text
     display self show_rows=10 format_terminal=False =
-        df = self.read max_rows=show_rows warn_if_more_rows=False
-        all_rows_count = self.row_count
-        display_dataframe df indices_count=0 all_rows_count format_terminal
+        data_fragment_with_warning = self.read max_rows=show_rows warn_if_more_rows=True
+        has_more_rows = data_fragment_with_warning.has_warnings warning_type=Not_All_Rows_Downloaded
+        data_fragment_cleared = data_fragment_with_warning.remove_warnings Not_All_Rows_Downloaded
+        # `row_count` means another Database query is performed, so we only do it if we need to.
+        all_rows_count = if has_more_rows then self.row_count else data_fragment_cleared.row_count
+        Display_Helpers.display_table table=data_fragment_cleared add_row_index=False max_rows_to_show=show_rows all_rows_count=all_rows_count format_terminal=format_terminal
 
     ## PRIVATE
        ADVANCED
@@ -2941,37 +2943,6 @@ make_table connection table_name columns ctx on_problems =
         problem_builder.attach_problems_before on_problems <|
             DB_Table.Value table_name connection cols ctx
 
-## PRIVATE
-
-   Renders an ASCII-art representation for a Table from a dataframe that
-   contains a fragment of the underlying data and count of all rows.
-
-   Arguments:
-   - df: The materialized dataframe that contains the data to be displayed, it
-     should have no indices set.
-   - indices_count: Indicates how many columns from the materialized dataframe
-     should be treated as indices in the display (index columns will be bold if
-     `format_terminal` is enabled).
-   - all_rows_count: The count of all rows in the underlying Table; if
-     `all_rows_count` is bigger than the amount of rows of `df`, an additional
-     line will be included that will say how many hidden rows there are.
-   - format_term: A boolean flag, specifying whether to use ANSI escape codes
-     for rich formatting in the terminal.
-display_dataframe : Table -> Integer -> Integer -> Boolean -> Text
-display_dataframe df indices_count all_rows_count format_terminal =
-    cols = Vector.from_polyglot_array df.java_table.getColumns
-    col_names = cols.map .getName . map normalize_string_for_display
-    col_vals = cols.map .getStorage
-    display_rows = df.row_count
-    rows = Vector.new display_rows row_num->
-        col_vals.map col->
-            if col.isNothing row_num then "Nothing" else get_item_string col row_num
-    table = print_table col_names rows indices_count format_terminal
-    if display_rows == all_rows_count then table else
-        missing_rows_count = all_rows_count - display_rows
-        missing = '\n\u2026 and ' + missing_rows_count.to_text + ' hidden rows.'
-        table + missing
-
 ## PRIVATE
    By default, join on the first column, unless it's a cross join, in which
    case there are no join conditions.

distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Connection.enso

@@ -52,7 +52,7 @@ type Postgres_Connection
        Arguments:
        - connection: the underlying connection.
        - make_new: a function that returns a new connection.
-    Value connection make_new
+    private Value connection make_new
 
     ## ICON data_input
        Closes the connection releasing the underlying database resources
@@ -320,4 +320,3 @@ parse_postgres_encoding encoding_name =
         fallback.catch Any _->
             warning = Unsupported_Database_Encoding.Warning "The database is using an encoding ("+encoding_name.to_display_text+") that is currently not supported by Enso. Falling back to UTF-8. Column/table names may not be mapped correctly if they contain unsupported characters."
             Warning.attach warning Encoding.utf_8
-

distribution/lib/Standard/Table/0.0.0-dev/src/Column.enso

@@ -30,7 +30,6 @@ import project.Value_Type.Value_Type
 from project.Errors import Conversion_Failure, Floating_Point_Equality, Inexact_Type_Coercion, Invalid_Column_Names, Invalid_Value_Type, No_Index_Set_Error
 from project.Internal.Column_Format import all
 from project.Internal.Java_Exports import make_date_builder_adapter, make_string_builder
-from project.Table import print_table
 
 polyglot java import org.enso.base.Time_Utils
 polyglot java import org.enso.table.data.column.operation.cast.CastProblemAggregator
@@ -155,19 +154,7 @@ type Column
              example_display = Examples.integer_column.display
     display : Integer -> Boolean -> Text
     display self show_rows=10 format_terminal=False =
-        java_col = self.java_column
-        col_name = normalize_string_for_display java_col.getName
-        storage = java_col.getStorage
-        num_rows = java_col.getSize
-        display_rows = num_rows.min show_rows
-        items = Vector.new display_rows num->
-            row = if storage.isNothing num then "Nothing" else
-                get_item_string storage num
-            [num.to_text, row]
-        table = print_table ["", col_name] items 1 format_terminal
-        if num_rows - display_rows <= 0 then table else
-            missing = '\n\u2026 and ' + (num_rows - display_rows).to_text + ' hidden rows.'
-            table + missing
+        self.to_table.display show_rows format_terminal
 
     ## PRIVATE
        ADVANCED