Google Sheets and Snowflake integration with OAuth credentials

build.sbt

@@ -607,11 +607,11 @@ val jline = Seq(
 )
 
 // === Google =================================================================
-val googleApiClientVersion         = "2.2.0"
-val googleApiServicesSheetsVersion = "v4-rev612-1.25.0"
-val googleAnalyticsAdminVersion    = "0.62.0"
-val googleAnalyticsDataVersion     = "0.63.0"
-val grpcVersion                    = "1.67.1"
+val googleApiClientVersion         = "2.7.1"
+val googleApiServicesSheetsVersion = "v4-rev20250106-2.0.0"
+val googleAnalyticsAdminVersion    = "0.66.0"
+val googleAnalyticsDataVersion     = "0.67.0"
+val grpcVersion                    = "1.69.0"
 
 // === Other ==================================================================
 
@@ -5020,7 +5020,7 @@ lazy val `std-google-api` = project
       "com.google.apis"       % "google-api-services-sheets" % googleApiServicesSheetsVersion exclude ("com.google.code.findbugs", "jsr305"),
       "com.google.analytics"  % "google-analytics-admin"     % googleAnalyticsAdminVersion exclude ("com.google.code.findbugs", "jsr305"),
       "com.google.analytics"  % "google-analytics-data"      % googleAnalyticsDataVersion exclude ("com.google.code.findbugs", "jsr305"),
-      "io.grpc"               % "grpc-netty-shaded"          % grpcVersion
+      "io.grpc"               % "grpc-netty-shaded"          % grpcVersion exclude ("com.google.code.findbugs", "jsr305")
     ),
     // Extract native libraries from grpc-netty-shaded-***.jar, and put them under
     // Standard/Google_Api/polyglot/lib directory. The minimized jar will
@@ -5063,6 +5063,7 @@ lazy val `std-google-api` = project
       .dependsOn(cleanPolyglotRoot)
       .value
   )
+  .dependsOn(`std-base` % "provided")
   .dependsOn(`std-table` % "provided")
 
 lazy val `std-database` = project

build_tools/cli/src/lib.rs

@@ -385,6 +385,9 @@ impl Processor {
                             config.add_standard_library_test_selection(
                                 StandardLibraryTestsSelection::Selected(vec![
                                     "Base_Tests".to_string(),
+                                    // Base Internal tests contain some cloud tests that need
+                                    // access to cloud internals
+                                    "Base_Internal_Tests".to_string(),
                                     // Table tests check integration of e.g. Postgres datalinks
                                     "Table_Tests".to_string(),
                                     // AWS tests check copying between Cloud and S3

distribution/lib/Standard/Base/0.0.0-dev/docs/api/Enso_Cloud/Enso_Secret.md

@@ -12,9 +12,11 @@
     - to_text self -> Standard.Base.Any.Any
 - type Enso_Secret
     - create name:Standard.Base.Data.Text.Text value:Standard.Base.Data.Text.Text parent:(Standard.Base.Enso_Cloud.Enso_File.Enso_File|Standard.Base.Nothing.Nothing)= -> Standard.Base.Any.Any
+    - credential_service_name self -> (Standard.Base.Data.Text.Text|Standard.Base.Nothing.Nothing)
     - delete self -> Standard.Base.Any.Any
     - exists name:Standard.Base.Data.Text.Text parent:(Standard.Base.Enso_Cloud.Enso_File.Enso_File|Standard.Base.Nothing.Nothing)= -> Standard.Base.Any.Any
     - get name:Standard.Base.Data.Text.Text parent:(Standard.Base.Enso_Cloud.Enso_File.Enso_File|Standard.Base.Nothing.Nothing)= -> Standard.Base.Any.Any
+    - is_credential self -> Standard.Base.Data.Boolean.Boolean
     - list parent:(Standard.Base.Enso_Cloud.Enso_File.Enso_File|Standard.Base.Nothing.Nothing)= -> Standard.Base.Any.Any
     - name self -> Standard.Base.Any.Any
     - path self -> Standard.Base.Any.Any
@@ -28,6 +30,7 @@
 - type Enso_Secret_Error
     - Access_Denied
     - to_display_text self -> Standard.Base.Any.Any
+- as_credential_reference secret:Standard.Base.Enso_Cloud.Enso_Secret.Enso_Secret -> Standard.Base.Enso_Cloud.Enso_Secret.CredentialReference
 - as_hideable_value value:(Standard.Base.Data.Text.Text|Standard.Base.Enso_Cloud.Enso_Secret.Enso_Secret|Standard.Base.Enso_Cloud.Enso_Secret.Derived_Secret_Value) -> Standard.Base.Any.Any
 - secret_asset_uri secret:Standard.Base.Any.Any -> Standard.Base.Any.Any
 - secret_resource_uri secret:Standard.Base.Any.Any -> Standard.Base.Any.Any

distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Enso_Secret.enso

@@ -6,6 +6,7 @@ import project.Data.Vector.Vector
 import project.Enso_Cloud.Enso_File.Enso_Asset_Type
 import project.Enso_Cloud.Enso_File.Enso_File
 import project.Enso_Cloud.Internal.Enso_Path.Enso_Path
+import project.Enso_Cloud.Internal.Existing_Enso_Asset.Credential_Secret_Metadata
 import project.Enso_Cloud.Internal.Existing_Enso_Asset.Existing_Enso_Asset
 import project.Enso_Cloud.Internal.Utils
 import project.Error.Error
@@ -15,6 +16,7 @@ import project.Network.HTTP.HTTP
 import project.Network.HTTP.HTTP_Method.HTTP_Method
 import project.Network.URI.URI
 import project.Nothing.Nothing
+import project.Runtime
 import project.Runtime.Context
 from project.Data.Boolean import Boolean, False, True
 from project.Data.Text.Extensions import all
@@ -27,11 +29,12 @@ polyglot java import org.enso.base.enso_cloud.HideableValue.Base64EncodeValue
 polyglot java import org.enso.base.enso_cloud.HideableValue.ConcatValues
 polyglot java import org.enso.base.enso_cloud.HideableValue.PlainValue
 polyglot java import org.enso.base.enso_cloud.HideableValue.SecretValue
+polyglot java import org.enso.base.enso_cloud.ExternalLibraryCredentialHelper.CredentialReference
 
 ## A reference to a secret stored in the Enso Cloud.
 type Enso_Secret
     ## PRIVATE
-    private Value internal_name:Text id:Text internal_path:Enso_Path
+    private Value internal_name:Text id:Text internal_path:Enso_Path credential_metadata:Credential_Secret_Metadata|Nothing
 
     ## GROUP Metadata
        ICON metadata
@@ -69,7 +72,9 @@ type Enso_Secret
                         Error.throw (Illegal_Argument.Error message)
                     error_handlers = Dictionary.from_vector [["resource_already_exists", handle_already_exists]]
                     id = Utils.http_request_as_json HTTP_Method.Post Utils.secrets_api body error_handlers=error_handlers
-                    Enso_Secret.Value name id path
+                    # A secret created using the regular flow is a normal secret, so it does not contain credential metadata
+                    credential_metadata = Nothing
+                    Enso_Secret.Value name id path credential_metadata
 
     ## GROUP Output
        ICON trash
@@ -93,7 +98,7 @@ type Enso_Secret
         effective_parent = parent.if_nothing Enso_File.current_working_directory
         secrets_as_assets = list_assets effective_parent . filter f-> f.asset_type == Enso_Asset_Type.Secret
         secrets_as_assets.map asset->
-            Enso_Secret.Value asset.title asset.id (effective_parent.enso_path.resolve asset.title)
+            _from_asset asset (effective_parent.enso_path.resolve asset.title)
 
     ## GROUP EnsoCloud
        ICON key
@@ -125,7 +130,7 @@ type Enso_Secret
         asset = Existing_Enso_Asset.resolve_path path if_not_found=(Error.throw Not_Found)
         parsed_path = Enso_Path.parse path
         parsed_path.if_not_error <| if asset.asset_type != Enso_Asset_Type.Secret then Error.throw (Illegal_Argument.Error "The provided path points to "+asset.asset_type.to_text+", not a Secret.") else
-            Enso_Secret.Value asset.title asset.id parsed_path
+            _from_asset asset parsed_path
 
     ## GROUP EnsoCloud
        ICON metadata
@@ -162,18 +167,22 @@ type Enso_Secret
     ## PRIVATE
        Returns a text representation of the secret.
     to_text : Text
-    to_text self = "Enso_Secret " + self.path.to_text
+    to_text self =
+        "Enso_Secret" + (_credential_info_text self) + " " + self.path.to_text
 
     ## PRIVATE
        Returns a display text representation of the secret.
     to_display_text : Text
-    to_display_text self = "Enso_Secret {" + self.name + "}"
+    to_display_text self =
+        "Enso_Secret {" + self.name + (_credential_info_text self) + "}"
 
     ## PRIVATE
        Converts the secret to a JSON object.
     to_js_object : JS_Object
     to_js_object self =
-        JS_Object.from_pairs [["type", "Enso_Secret"], ["constructor", "get"], ["path", self.path.to_text]]
+        extra = if self.credential_metadata.is_nothing then [] else
+            [["credential_for", self.credential_metadata.service_name]]
+        JS_Object.from_pairs [["type", "Enso_Secret"], ["constructor", "get"], ["path", self.path.to_text]]+extra
 
     ## PRIVATE
        GROUP convert
@@ -182,6 +191,21 @@ type Enso_Secret
     pretty : Text
     pretty self = "Enso_Secret.get " + self.path.to_text.pretty
 
+    ## PRIVATE
+       Checks if this secret is a regular text value or a credential used for
+       connecting with external services.
+
+       Regular secrets and credentials can be used in different contexts.
+    is_credential self -> Boolean = self.credential_metadata.is_nothing.not
+
+    ## PRIVATE
+       Returns the name of the service that this credential is associated with, or Nothing if this secret is not a credential.
+
+       Useful in filtering credentials for widgets.
+    credential_service_name self -> Text | Nothing =
+        self.credential_metadata.if_not_nothing <|
+            self.credential_metadata.service_name
+
 ## PRIVATE
 type Enso_Secret_Error
     ## PRIVATE
@@ -214,8 +238,9 @@ type Derived_Secret_Value
     to_plain_text : Text ! Enso_Secret_Error
     to_plain_text self =
         java_repr = as_hideable_value self
-        if java_repr.containsSecrets then Error.throw Enso_Secret_Error.Access_Denied else
-            java_repr.safeResolve
+        if java_repr.containsSecrets then
+            Error.throw Enso_Secret_Error.Access_Denied
+        java_repr.safeResolve
 
     ## PRIVATE
     to_text : Text
@@ -240,16 +265,42 @@ Derived_Secret_Value.from (that : Enso_Secret) = Derived_Secret_Value.Secret_Val
 as_hideable_value : Text | Enso_Secret | Derived_Secret_Value -> HideableValue
 as_hideable_value (value : Text | Enso_Secret | Derived_Secret_Value) = case value of
     text : Text -> HideableValue.PlainValue.new text
-    secret : Enso_Secret -> HideableValue.SecretValue.new secret.id
+    secret : Enso_Secret ->
+        if secret.is_credential then
+            Error.throw (Illegal_Argument.Error "A credential for "+secret.credential_service_name+" cannot be used as a regular secret value. Credentials can only be used in their corresponding services.")
+        HideableValue.SecretValue.new secret.id
     Derived_Secret_Value.Plain_Text text -> as_hideable_value text
     Derived_Secret_Value.Secret_Value secret -> as_hideable_value secret
     Derived_Secret_Value.Concat left right -> HideableValue.ConcatValues.new (as_hideable_value left) (as_hideable_value right)
     Derived_Secret_Value.Base_64_Encode inner -> HideableValue.Base64EncodeValue.new (as_hideable_value inner)
 
+## PRIVATE
+   Used by library implementations to convert an Enso object into the Java counterpart.
+as_credential_reference (secret : Enso_Secret) -> CredentialReference =
+    case secret.credential_metadata of
+        Nothing ->
+            Error.throw (Illegal_Argument.Error "Secret "+secret.name+" is not a credential.")
+        metadata ->
+            CredentialReference.new secret.id metadata.service_name
+
 ## PRIVATE
 secret_resource_uri secret =
     Utils.secrets_api + "/" + secret.id
 
 ## PRIVATE
 secret_asset_uri secret =
     Utils.assets_api + "/" + secret.id
+
+private _from_asset (asset : Existing_Enso_Asset) (path : Enso_Path) -> Enso_Secret =
+    Runtime.assert (asset.metadata != Nothing)
+    Enso_Secret.Value asset.title asset.id path asset.metadata.credential_metadata
+
+private _credential_info_text (secret : Enso_Secret) -> Text =
+    case secret.credential_metadata of
+        Nothing -> ""
+        credential_metadata ->
+            state_suffix = case credential_metadata.state of
+                "Expired" -> " (expired)"
+                "WaitingForAuthentication" -> " (waiting for authentication)"
+                _ -> ""
+            " (credentials for "+credential_metadata.service_name+state_suffix+")"

distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Internal/Existing_Enso_Asset.enso

@@ -129,7 +129,7 @@ asset_type_from_id id:Text -> Enso_Asset_Type =
    The asset metadata that is currently automatically included as part of the path resolution response.
 type Asset_Metadata
     ## PRIVATE
-    Value modified_at:Date_Time description:Text|Nothing labels:Vector
+    Value modified_at:Date_Time description:Text|Nothing labels:Vector credential_metadata:Credential_Secret_Metadata|Nothing
 
     ## PRIVATE
     from_json json -> Asset_Metadata =
@@ -138,7 +138,21 @@ type Asset_Metadata
             . catch Time_Error error-> Error.throw (Enso_Cloud_Error.Invalid_Response_Payload error)
         description = get_optional_field "description" json expected_type=Text
         labels = get_optional_field "labels" json expected_type=Vector if_missing=[]
-        Asset_Metadata.Value modified_at description labels
+
+        credential_metadata_field = get_optional_field "credentialMetadata" json expected_type=JS_Object
+        credential_metadata = credential_metadata_field.if_not_nothing <|
+            service_name = get_required_field "serviceName" credential_metadata_field expected_type=Text
+            state = get_required_field "state" credential_metadata_field expected_type=Text
+            expiration_date_text = get_optional_field "expirationDate" credential_metadata_field expected_type=Text
+            expiration_date = expiration_date_text.if_not_nothing <|
+                Date_Time.parse expiration_date_text Date_Time_Formatter.iso_offset_date_time
+                    . catch Time_Error error-> Error.throw (Enso_Cloud_Error.Invalid_Response_Payload error)
+            Credential_Secret_Metadata.Value service_name state expiration_date
+        Asset_Metadata.Value modified_at description labels credential_metadata
+
+## PRIVATE
+type Credential_Secret_Metadata
+    Value service_name:Text state:Text expiration_date:Date_Time|Nothing
 
 ## PRIVATE
 type Asset_Cache

distribution/lib/Standard/Database/0.0.0-dev/docs/api/Internal/JDBC_Connection.md

@@ -17,8 +17,10 @@
     - with_prepared_statement self query:Standard.Base.Any.Any statement_setter:Standard.Base.Any.Any action:Standard.Base.Any.Any skip_log:Standard.Base.Any.Any= -> Standard.Base.Any.Any
 - close_connection connection:Standard.Base.Any.Any -> Standard.Base.Any.Any
 - create url:Standard.Base.Any.Any properties:Standard.Base.Any.Any -> Standard.Base.Any.Any
+- from_java java_jdbc_connection:Standard.Base.Any.Any -> Standard.Base.Any.Any
 - get_pragma_value jdbc_connection:Standard.Base.Any.Any sql:Standard.Base.Any.Any -> Standard.Base.Any.Any
 - handle_sql_errors ~action:Standard.Base.Any.Any related_query:Standard.Base.Any.Any= -> Standard.Base.Any.Any
 - log_sql_if_enabled jdbc_connection:Standard.Base.Any.Any ~query_text:Standard.Base.Any.Any -> Standard.Base.Any.Any
 - profile_sql_if_enabled jdbc_connection:Standard.Database.Internal.JDBC_Connection.JDBC_Connection ~query_text:Standard.Base.Data.Text.Text ~action:Standard.Base.Any.Any -> Standard.Base.Any.Any
+- properties_as_java_props properties:Standard.Base.Any.Any -> Standard.Base.Any.Any
 - set_statement_values stmt:Standard.Base.Any.Any statement_setter:Standard.Base.Any.Any values:Standard.Base.Any.Any expected_type_hints:Standard.Base.Any.Any= -> Standard.Base.Any.Any

distribution/lib/Standard/Database/0.0.0-dev/src/Internal/JDBC_Connection.enso

@@ -275,15 +275,22 @@ type JDBC_Connection
    - properties: A vector of properties for the connection.
 create : Text -> Vector -> JDBC_Connection
 create url properties = handle_sql_errors <|
-    java_props = properties.map pair->
+    java_connection = JDBCProxy.getConnection url (properties_as_java_props properties)
+    from_java java_connection
+
+## PRIVATE
+properties_as_java_props properties =
+    properties.map pair->
         # Some parameters may be passed by the dialect as a `HideableValue` directly, so they do not need to be converted.
         value = pair.second
         java_value = if value.is_a HideableValue then value else
             as_hideable_value value
         Java_Pair.create pair.first java_value
-    java_connection = JDBCProxy.getConnection url java_props
 
-    resource = Managed_Resource.register java_connection close_connection
+## PRIVATE
+   A special method used by dialects that construct a Java JDBC connection using some custom logic.
+from_java java_jdbc_connection =
+    resource = Managed_Resource.register java_jdbc_connection close_connection
     synchronizer = OperationSynchronizer.new
     JDBC_Connection.Value resource synchronizer