Enable audit logs for Snowflake backend

app/dashboard/src/data/__tests__/dataLinkSchema.test.ts

@@ -34,6 +34,7 @@ const REPO_ROOT = url.fileURLToPath(new URL('../'.repeat(DIR_DEPTH), import.meta
 const BASE_DATA_LINKS_ROOT = path.resolve(REPO_ROOT, 'test/Base_Tests/data/datalinks/')
 const S3_DATA_LINKS_ROOT = path.resolve(REPO_ROOT, 'test/AWS_Tests/data/')
 const TABLE_DATA_LINKS_ROOT = path.resolve(REPO_ROOT, 'test/Table_Tests/data/datalinks/')
+const SNOWFLAKE_DATA_LINKS_ROOT = path.resolve(REPO_ROOT, 'test/Snowflake_Tests/data/datalinks/')
 
 v.test('correctly validates example HTTP .datalink files with the schema', () => {
   const schemas = [
@@ -97,3 +98,11 @@ v.test('correctly validates example Database .datalink files with the schema', (
     testSchema(json, schema)
   }
 })
+
+v.test('correctly validates example Snowflake .datalink files with the schema', () => {
+  const schemas = ['snowflake-db.datalink']
+  for (const schema of schemas) {
+    const json = loadDataLinkFile(path.resolve(SNOWFLAKE_DATA_LINKS_ROOT, schema))
+    testSchema(json, schema)
+  }
+})

distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Internal/Authentication.enso

@@ -145,8 +145,7 @@ type Refresh_Token_Data
                 HTTP_Error.Status_Error status message _ ->
                     Authentication_Service.log_message level=..Warning "Refresh token request failed with status "+status.to_text+": "+(message.if_nothing "<no message>")+"."
 
-                    # As per OAuth specification, an expired refresh token should result in a 401 status code: https://www.rfc-editor.org/rfc/rfc6750.html#section-3.1
-                    if status.code == 401 then Panic.throw (Cloud_Session_Expired.Error error) else
+                    if is_refresh_token_expired status message then Panic.throw (Cloud_Session_Expired.Error error) else
                         # Otherwise, we fail with the generic error that gives more details.
                         Panic.throw (Enso_Cloud_Error.Connection_Error error)
                 _ -> Panic.throw (Enso_Cloud_Error.Connection_Error error)
@@ -175,6 +174,13 @@ type Refresh_Token_Data
    it to reduce the chance of it expiring during a request.
 token_early_refresh_period = Duration.new minutes=2
 
+## PRIVATE
+is_refresh_token_expired status message -> Boolean =
+    # As per OAuth specification, an expired refresh token should result in a 401 status code: https://www.rfc-editor.org/rfc/rfc6750.html#section-3.1
+    # But empirically, it failed with: 400 Bad Request: {"__type":"NotAuthorizedException","message":"Refresh Token has expired"}.
+    # Let's just handle both just in case
+    (status.code == 401) || (status.code == 400 && message.contains '"Refresh Token has expired"')
+
 ## PRIVATE
    A sibling to `get_required_field`.
    This one raises `Illegal_State` error, because it is dealing with local files and not cloud responses.

distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Data_Link/Postgres_Data_Link.enso

@@ -9,17 +9,17 @@ from Standard.Base.Enso_Cloud.Public_Utils import get_optional_field, get_requir
 import project.Connection.Connection_Options.Connection_Options
 import project.Connection.Credentials.Credentials
 import project.Connection.Postgres.Postgres
+import project.Internal.DB_Data_Link_Helpers
 
 ## PRIVATE
 type Postgres_Data_Link
     ## PRIVATE
        A data-link returning a connection to the specified database.
-    Connection details:Postgres related_asset_id:Text|Nothing
+    Connection details:Postgres source:Data_Link_Source_Metadata
 
     ## PRIVATE
        A data-link returning a query to a specific table within a database.
-    Table name:Text details:Postgres related_asset_id:Text|Nothing
-
+    Table name:Text details:Postgres source:Data_Link_Source_Metadata
 
     ## PRIVATE
     parse json source:Data_Link_Source_Metadata -> Postgres_Data_Link =
@@ -34,23 +34,18 @@ type Postgres_Data_Link
             password = get_required_field "password" credentials_json |> parse_secure_value
             Credentials.Username_And_Password username password
 
-        related_asset_id = case source of
-            Data_Link_Source_Metadata.Cloud_Asset id -> id
-            _ -> Nothing
         details = Postgres.Server host=host port=port database=db_name schema=schema credentials=credentials
         case get_optional_field "table" json expected_type=Text of
             Nothing ->
-                Postgres_Data_Link.Connection details related_asset_id
+                Postgres_Data_Link.Connection details source
             table_name : Text ->
-                Postgres_Data_Link.Table table_name details related_asset_id
+                Postgres_Data_Link.Table table_name details source
 
     ## PRIVATE
     read self (format = Auto_Detect) (on_problems : Problem_Behavior) =
         _ = on_problems
         if format != Auto_Detect then Error.throw (Illegal_Argument.Error "Only Auto_Detect can be used with a Postgres Data Link, as it points to a database.") else
-            audit_mode = if Enso_User.is_logged_in then "cloud" else "local"
-            options_vector = [["enso.internal.audit", audit_mode]] + (if self.related_asset_id.is_nothing then [] else [["enso.internal.relatedAssetId", self.related_asset_id]])
-            default_options = Connection_Options.Value options_vector
+            default_options = DB_Data_Link_Helpers.data_link_connection_parameters self.source
             connection = self.details.connect default_options allow_data_links=False
             case self of
                 Postgres_Data_Link.Connection _ _ -> connection

distribution/lib/Standard/Database/0.0.0-dev/src/Internal/DB_Data_Link_Helpers.enso

@@ -0,0 +1,13 @@
+from Standard.Base import all
+
+from Standard.Base.Enso_Cloud.Data_Link_Helpers import Data_Link_Source_Metadata
+
+import project.Connection.Connection_Options.Connection_Options
+
+data_link_connection_parameters (source : Data_Link_Source_Metadata) -> Connection_Options =
+    related_asset_id = case source of
+        Data_Link_Source_Metadata.Cloud_Asset id -> id
+        _ -> Nothing
+    audit_mode = if Enso_User.is_logged_in then "cloud" else "local"
+    options_vector = [["enso.internal.audit", audit_mode]] + (if related_asset_id.is_nothing then [] else [["enso.internal.relatedAssetId", related_asset_id]])
+    Connection_Options.Value options_vector

distribution/lib/Standard/Snowflake/0.0.0-dev/src/Connection/Snowflake_Details.enso

@@ -36,8 +36,10 @@ type Snowflake_Details
 
        Arguments:
        - options: Overrides for the connection properties.
-    connect : Connection_Options -> Snowflake_Connection
-    connect self options =
+    connect : Connection_Options -> Boolean -> Snowflake_Connection
+    connect self options (allow_data_links : Boolean = True) =
+        # TODO use this once #11294 is done
+        _ = allow_data_links
         properties = options.merge self.jdbc_properties
 
         ## Cannot use default argument values as gets in an infinite loop if you do.

distribution/lib/Standard/Snowflake/0.0.0-dev/src/Snowflake_Data_Link.enso

@@ -1,26 +1,25 @@
 from Standard.Base import all
 import Standard.Base.Errors.Illegal_Argument.Illegal_Argument
-from Standard.Base.Enso_Cloud.Data_Link_Helpers import parse_secure_value
+from Standard.Base.Enso_Cloud.Data_Link_Helpers import Data_Link_Source_Metadata, parse_secure_value
 from Standard.Base.Enso_Cloud.Public_Utils import get_optional_field, get_required_field
 
-import Standard.Database.Connection.Connection_Options.Connection_Options
 import Standard.Database.Connection.Credentials.Credentials
+import Standard.Database.Internal.DB_Data_Link_Helpers
 
 import project.Connection.Snowflake_Details.Snowflake_Details
 
 ## PRIVATE
 type Snowflake_Data_Link
     ## PRIVATE
        A data-link returning a connection to the specified database.
-    Connection details:Snowflake_Details
+    Connection details:Snowflake_Details source:Data_Link_Source_Metadata
 
     ## PRIVATE
        A data-link returning a query to a specific table within a database.
-    Table name:Text details:Snowflake_Details
+    Table name:Text details:Snowflake_Details source:Data_Link_Source_Metadata
 
     ## PRIVATE
     parse json source -> Snowflake_Data_Link =
-        _ = source
         account = get_required_field "account" json expected_type=Text
         db_name = get_required_field "database_name" json expected_type=Text
         schema = get_optional_field "schema" json if_missing="SNOWFLAKE" expected_type=Text
@@ -34,17 +33,17 @@ type Snowflake_Data_Link
         details = Snowflake_Details.Snowflake account=account database=db_name schema=schema warehouse=warehouse credentials=credentials
         case get_optional_field "table" json expected_type=Text of
             Nothing ->
-                Snowflake_Data_Link.Connection details
+                Snowflake_Data_Link.Connection details source
             table_name : Text ->
-                Snowflake_Data_Link.Table table_name details
+                Snowflake_Data_Link.Table table_name details source
 
     ## PRIVATE
     read self (format = Auto_Detect) (on_problems : Problem_Behavior) =
         _ = on_problems
         if format != Auto_Detect then Error.throw (Illegal_Argument.Error "Only Auto_Detect can be used with a Snowflake Data Link, as it points to a database.") else
-            default_options = Connection_Options.Value
-            connection = self.details.connect default_options
+            default_options = DB_Data_Link_Helpers.data_link_connection_parameters self.source
+            connection = self.details.connect default_options allow_data_links=False
             case self of
-                Snowflake_Data_Link.Connection _ -> connection
-                Snowflake_Data_Link.Table table_name _ ->
+                Snowflake_Data_Link.Connection _ _ -> connection
+                Snowflake_Data_Link.Table table_name _ _ ->
                     connection.query table_name

test/Snowflake_Tests/data/datalinks/snowflake-db.datalink

@@ -0,0 +1,12 @@
+{
+  "type": "Snowflake_Connection",
+  "libraryName": "Standard.Snowflake",
+  "account": "ACCOUNT",
+  "database_name": "DBNAME",
+  "schema": "SCHEMA",
+  "warehouse": "WAREHOUSE",
+  "credentials": {
+    "username": "USERNAME",
+    "password": "PASSWORD"
+  }
+}

test/Snowflake_Tests/src/Snowflake_Spec.enso

@@ -1,7 +1,9 @@
 from Standard.Base import all
+import Standard.Base.Enso_Cloud.Data_Link.Data_Link
 import Standard.Base.Errors.Illegal_Argument.Illegal_Argument
 import Standard.Base.Errors.Illegal_State.Illegal_State
 import Standard.Base.Runtime.Ref.Ref
+from Standard.Base.Enso_Cloud.Data_Link_Helpers import secure_value_to_json
 
 from Standard.Table import Table, Value_Type, Aggregate_Column, Bits, expr
 from Standard.Table.Errors import Invalid_Column_Names, Inexact_Type_Coercion, Duplicate_Output_Column_Names
@@ -20,6 +22,7 @@ from Standard.Test import all
 import Standard.Test.Test_Environment
 
 import enso_dev.Table_Tests
+import enso_dev.Table_Tests.Database.Common.Audit_Spec
 import enso_dev.Table_Tests.Database.Common.Common_Spec
 import enso_dev.Table_Tests.Database.Common.IR_Spec
 import enso_dev.Table_Tests.Database.Transaction_Spec
@@ -567,16 +570,43 @@ supported_replace_params =
     e4 = [Replace_Params.Value DB_Column Case_Sensitivity.Default False, Replace_Params.Value DB_Column Case_Sensitivity.Sensitive False]
     Hashset.from_vector <| e0 + e1 + e2 + e3 + e4
 
+transform_file base_file connection_details =
+    content = Data_Link.read_raw_config base_file
+
+    if connection_details.credentials.password.is_a Enso_Secret then
+        Panic.throw (Illegal_State.Error "Currently Snowflake tests need a plaintext password in environment variable to run.")
+
+    new_content = content
+        . replace "ACCOUNT" connection_details.account
+        . replace "DBNAME" connection_details.database
+        . replace "SCHEMA" connection_details.schema
+        . replace "WAREHOUSE" connection_details.warehouse
+        . replace "USERNAME" connection_details.credentials.username
+        . replace "PASSWORD" connection_details.credentials.password
+
+    temp_file = File.create_temporary_file "snowflake-test-db" ".datalink"
+    Data_Link.write_raw_config temp_file new_content replace_existing=True . if_not_error temp_file
+
+type Temporary_Data_Link_File
+    Value ~get
+
+    make connection_details = Temporary_Data_Link_File.Value <|
+        transform_file (enso_project.data / "datalinks" / "snowflake-db.datalink") connection_details
+
 add_table_specs suite_builder =
-    case create_connection_builder of
+    case get_configured_connection_details of
         Nothing ->
             message = "Snowflake test connection is not configured. See README.md for instructions."
             suite_builder.group "[Snowflake] Database tests" pending=message (_-> Nothing)
-        connection_builder ->
-            db_name = get_configured_connection_details.database
+        connection_details ->
+            db_name = connection_details.database
+            connection_builder = _ -> Database.connect connection_details
             add_snowflake_specs suite_builder connection_builder db_name
             Transaction_Spec.add_specs suite_builder connection_builder "[Snowflake] "
 
+            data_link_file = Temporary_Data_Link_File.make connection_details
+            Audit_Spec.add_specs suite_builder "[Snowflake] " data_link_file.get database_pending=Nothing
+
             suite_builder.group "[Snowflake] Secrets in connection settings" group_builder->
                 cloud_setup = Cloud_Tests_Setup.prepare
                 base_details = get_configured_connection_details
@@ -623,14 +653,6 @@ get_configured_connection_details = Panic.rethrow <|
         credentials = Credentials.Username_And_Password user resolved_password
         Snowflake_Details.Snowflake account_name credentials database schema warehouse
 
-## Returns a function that takes anything and returns a new connection.
-   The function creates a _new_ connection on each invocation
-   (this is needed for some tests that need multiple distinct connections).
-create_connection_builder =
-    connection_details = get_configured_connection_details
-    connection_details.if_not_nothing <|
-        _ -> Database.connect connection_details
-
 add_specs suite_builder =
     add_table_specs suite_builder