OAuth flow prototypes

build.sbt

@@ -603,11 +603,11 @@ val jline = Seq(
 )
 
 // === Google =================================================================
-val googleApiClientVersion         = "2.2.0"
-val googleApiServicesSheetsVersion = "v4-rev612-1.25.0"
-val googleAnalyticsAdminVersion    = "0.62.0"
-val googleAnalyticsDataVersion     = "0.63.0"
-val grpcVersion                    = "1.67.1"
+val googleApiClientVersion         = "2.7.1"
+val googleApiServicesSheetsVersion = "v4-rev20250106-2.0.0"
+val googleAnalyticsAdminVersion    = "0.66.0"
+val googleAnalyticsDataVersion     = "0.67.0"
+val grpcVersion                    = "1.69.0"
 
 // === Other ==================================================================
 
@@ -5037,6 +5037,7 @@ lazy val `std-google-api` = project
       .dependsOn(cleanPolyglotRoot)
       .value
   )
+  .dependsOn(`std-base` % "provided")
   .dependsOn(`std-table` % "provided")
 
 lazy val `std-database` = project

build_tools/build/src/cloud_tests/mod.rs

@@ -33,6 +33,11 @@ pub async fn prepare_credentials_file(auth_config: AuthConfig) -> Result<NamedTe
     Ok(credentials_temp_file)
 }
 
+pub async fn build_credentials_file(auth_config: AuthConfig, path: &Path) -> Result<()> {
+    let credentials = build_credentials(auth_config).await?;
+    save_credentials(&credentials, path)
+}
+
 #[derive(Debug)]
 pub struct AuthConfig {
     web_client_id: String,

build_tools/cli/src/arg/backend.rs

@@ -59,6 +59,9 @@ pub enum Command {
     CiCheck {},
     /// Perform the stdlib API checks
     StdlibApiCheck {},
+
+    /// Generate Cloud credentials
+    GenerateCloudCredentials {},
 }
 
 #[derive(Args, Clone, Debug, PartialEq)]

build_tools/cli/src/lib.rs

@@ -29,6 +29,7 @@ use crate::arg::WatchJob;
 use anyhow::Context;
 use arg::BuildDescription;
 use clap::Parser;
+use enso_build::cloud_tests;
 use enso_build::config::Config;
 use enso_build::context::BuildContext;
 use enso_build::engine::context::EnginePackageProvider;
@@ -449,6 +450,12 @@ impl Processor {
                 .void_ok()
                 .boxed()
             }
+            arg::backend::Command::GenerateCloudCredentials {} => async move {
+                let auth_config = cloud_tests::build_auth_config_from_environment()?;
+                let path = Path::new("enso.credentials");
+                cloud_tests::build_credentials_file(auth_config, path).await
+            }
+            .boxed(),
         }
     }
 

distribution/lib/Standard/Google_Api/0.0.0-dev/src/Google_Sheets.enso

@@ -3,6 +3,7 @@ import Standard.Base.Data.Array_Proxy.Array_Proxy
 import Standard.Base.Metadata.Display
 from Standard.Base.Metadata import make_single_choice, Widget
 from Standard.Base.Metadata.Widget import Single_Choice, Vector_Editor
+from Standard.Base.Enso_Cloud.Enso_Secret import as_hideable_value
 
 from Standard.Table import Table
 
@@ -11,8 +12,12 @@ polyglot java import com.google.api.client.googleapis.javanet.GoogleNetHttpTrans
 polyglot java import com.google.api.client.json.gson.GsonFactory
 polyglot java import com.google.api.services.sheets.v4.Sheets
 polyglot java import com.google.api.services.sheets.v4.SheetsScopes
+polyglot java import java.io.IOException
 polyglot java import java.util.Collections
 
+polyglot java import org.enso.google.GoogleOAuthSecretReader
+polyglot java import org.enso.google.GoogleSheetsHelpers
+
 type Google_Sheets
     ## PRIVATE
     Service java_service
@@ -21,18 +26,21 @@ type Google_Sheets
        Initializes the Google Sheets instance using the given credentials file.
 
        Arguments:
-       - secret_file: a file containing Google Service Account credentials to use to
+       - secret_file: a Cloud secret or a file containing Google Service Account credentials to use to
          access Google services. The credentials file can be downloaded from the
          Google Admin Console when generating a key.
-    initialize : File -> Google_Sheets
+    initialize : File|Enso_Secret -> Google_Sheets
     initialize secret_file =
-        app_name = 'Enso'
-        credential = secret_file.with_input_stream [File_Access.Read] stream->
-            stream.with_java_stream is->
-                GoogleCredential.fromStream is . createScoped (Collections.singleton SheetsScopes.SPREADSHEETS)
-        http_transport = GoogleNetHttpTransport.newTrustedTransport
-        json_factory = GsonFactory.getDefaultInstance
-        service = Sheets.Builder.new http_transport json_factory credential . setApplicationName app_name . build
+        base_credential = case secret_file of
+            secret : Enso_Secret ->
+                # TODO to avoid passing credentials around, prod should probably create instance of Sheets and return it here
+                GoogleOAuthSecretReader.createCredentialFromSecretValue (as_hideable_value secret)
+            _ ->
+                secret_file.with_input_stream [File_Access.Read] stream->
+                    stream.with_java_stream is->
+                        GoogleCredential.fromStream is
+        credential = base_credential.createScoped (Collections.singleton SheetsScopes.SPREADSHEETS)
+        service = GoogleSheetsHelpers.createService credential
         Google_Sheets.Service service
 
     ## ICON data_input
@@ -45,8 +53,19 @@ type Google_Sheets
          `'Sheet1!A1:B7'`.
     get_table : Text -> Text -> Table
     get_table self sheet_id sheet_range =
-        request = self.java_service.spreadsheets.values.get sheet_id sheet_range . setMajorDimension 'COLUMNS' . setValueRenderOption 'UNFORMATTED_VALUE'
-        response = request.execute
-        values = Vector.from_polyglot_array response.getValues
-        columned = values.map v-> [v.first, v.drop 1]
-        Table.new columned
+        values = _handle_exceptions <|
+            GoogleSheetsHelpers.getSheetRange self.java_service sheet_id sheet_range
+        # The first entry is the column name (A, B, ...) and the rest are the values.
+        column_vectors = values.map v-> [v.first, v.drop 1]
+        # The column lengths may not necessarily be the same, but Table.new expects the same lengths, so we normalize:
+        row_count = column_vectors.map (p-> p.second.length) . compute ..Maximum
+        normalized = column_vectors.map p->
+            [p.first, p.second.pad row_count Nothing]
+        Table.new normalized
+
+type Google_Api_Error
+    Error message cause
+
+private _handle_exceptions ~action =
+    Panic.catch IOException action caught_panic->
+        Error.throw (Google_Api_Error.Error caught_panic.payload.to_text caught_panic.payload)

distribution/lib/Standard/Google_Api/0.0.0-dev/src/OAuth_Prototype.enso

@@ -0,0 +1,74 @@
+from Standard.Base import all
+import Standard.Base.Data.Base_64.Base_64
+import Standard.Base.Errors.Common.Not_Found
+import Standard.Base.Errors.Illegal_State.Illegal_State
+import Standard.Base.Network.HTTP.Request_Body.Request_Body
+import Standard.Base.Runtime.Managed_Resource.Managed_Resource
+
+polyglot java import org.enso.base.oauth.OAuthCallback
+
+authenticate_service scopes:Vector secret_name:Text location:Enso_File=Enso_File.home -> Enso_Secret =
+    uri = create_authorization_uri scopes
+    result = Managed_Resource.bracket (OAuthCallback.createCallbackServer 51234) (.close) server->
+        # We start the server first to ensure we can 'reserve' the port before opening the browser
+        IO.println "Please open "+uri.to_text+" in your browser and follow the instructions."
+        server.waitForCallback
+    code = OAuth_Callback_Response.decode result . code
+
+    refresh_token = exchange_one_time_code_for_refresh_token code
+    secret = Enso_Secret.create secret_name (make_secret_payload refresh_token) parent=location
+    Enso_File.new secret.path . add_label "Credentials"
+    secret
+
+
+exchange_one_time_code_for_refresh_token code:Text -> Text =
+    uri = create_token_uri
+    params = Dictionary.from_vector [["grant_type", "authorization_code"], ["code", code], ["redirect_uri", redirect_uri], ["client_id", client_id], ["client_secret", client_secret]]
+    request_body = Request_Body.Form_Data params url_encoded=True
+    response = Data.post uri body=request_body response_format=JSON_Format
+    IO.println response
+    response.get "refresh_token"
+
+make_secret_payload refresh_token:Text -> Text =
+    JS_Object.from_pairs [["type", "authorized_user"], ["refresh_token", refresh_token], ["client_id", client_id], ["client_secret", client_secret]]
+        . to_json
+
+create_authorization_uri scopes:Vector -> URI =
+    if scopes.is_empty then
+        Panic.throw (Illegal_State.Error "No scopes provided.")
+    URI.from "https://accounts.google.com/o/oauth2/v2/auth"
+        . add_query_argument "response_type" "code"
+        . add_query_argument "access_type" "offline"
+        . add_query_argument "client_id" client_id
+        . add_query_argument "redirect_uri" redirect_uri
+        . add_query_argument "scope" (scopes.join " ")
+        # Add prompt=consent to ensure that refresh token is returned always, not only first time...
+        . add_query_argument "prompt" "consent"
+        . add_query_argument "state" "foobar"
+
+create_token_uri -> URI =
+    URI.from "https://oauth2.googleapis.com/token"
+
+get_env_var name:Text -> Text =
+    Environment.get name if_missing=(Panic.throw (Illegal_State.Error "Environment variable not set: "+name))
+client_id = get_env_var "GOOGLE_APP_CLIENT_ID"
+client_secret = get_env_var "GOOGLE_APP_CLIENT_SECRET"
+redirect_uri = "http://localhost:51234/oauth"
+
+type OAuth_Callback_Response
+    Value code:Text state:Text|Nothing
+
+    decode response:Text -> OAuth_Callback_Response =
+        parts = response.split "&"
+        code_prefix = "code="
+        state_prefix = "state="
+        code = parts
+            . find (e-> e.starts_with code_prefix)
+            . drop code_prefix.length
+            . catch Not_Found _->
+                Error.throw (Illegal_State.Error "Malformed OAuth response: No code found.")
+        state = parts
+            . find (e-> e.starts_with state_prefix)
+            . drop state_prefix.length
+            . catch Not_Found _->Nothing
+        OAuth_Callback_Response.Value code state

distribution/lib/Standard/Snowflake/0.0.0-dev/src/Connection/Snowflake_Details.enso

@@ -15,6 +15,8 @@ import Standard.Database.Internal.Data_Link_Setup.Data_Link_Setup
 import project.Connection.Key_Pair_Credentials.Key_Pair_Credentials
 import project.Connection.Widgets
 import project.Snowflake_Connection.Snowflake_Connection
+import project.OAuth_Test.Access_Token
+import project.OAuth_Test.Refresh_Token
 
 polyglot java import org.enso.base.enso_cloud.InterpretAsPrivateKey
 polyglot java import net.snowflake.client.jdbc.SnowflakeDriver
@@ -30,7 +32,7 @@ type Snowflake_Details
        - warehouse: The name of the warehouse to use.
     @account (Text_Input display=..Always)
     @credentials Widgets.password_or_keypair_widget
-    Snowflake (account:Text=(Missing_Argument.throw "account")) (credentials:Credentials|Key_Pair_Credentials=(Missing_Argument.throw "credentials")) database:Text="SNOWFLAKE" schema:Text="PUBLIC" warehouse:Text=""
+    Snowflake (account:Text=(Missing_Argument.throw "account")) (credentials:Credentials|Key_Pair_Credentials|Access_Token|Refresh_Token=(Missing_Argument.throw "credentials")) database:Text="SNOWFLAKE" schema:Text="PUBLIC" warehouse:Text=""
 
     ## PRIVATE
        Attempt to resolve the constructor.
@@ -86,6 +88,11 @@ type Snowflake_Details
                         secret_as_private_key = InterpretAsPrivateKey.new (as_hideable_value secret)
                         [Pair.new 'privateKey' secret_as_private_key]
                 [Pair.new 'user' username] + key_part
+            token : Access_Token ->
+                _keys_for_access_token token
+            refresh_token : Refresh_Token ->
+                access_token = refresh_token.exchange_for_access_token
+                _keys_for_access_token access_token
 
         database = [Pair.new 'db' self.database]
         schema = [Pair.new 'schema' self.schema]
@@ -123,3 +130,8 @@ private _enhance_connection_errors (details:Snowflake_Details) ~action =
             Error.throw (Illegal_State.Error "The provided private key file is not supported by the Snowflake driver. We recommend generating the key using `generate_key_pair` method and storing it as a Secret in Enso Cloud. The original error was: "+message cause=error)
 
         result
+
+private _keys_for_access_token (token : Access_Token) -> Vector =
+    if Date_Time.now > token.expiry then
+        Error.throw (Illegal_State.Error "Access token has expired on "+token.expiry.to_display_text)
+    [Pair.new 'authenticator' 'oauth', Pair.new 'user' token.username, Pair.new 'token' token.token]

distribution/lib/Standard/Snowflake/0.0.0-dev/src/OAuth_Test.enso

@@ -0,0 +1,118 @@
+from Standard.Base import all
+import Standard.Base.Data.Base_64.Base_64
+import Standard.Base.Errors.Common.Not_Found
+import Standard.Base.Errors.Illegal_State.Illegal_State
+import Standard.Base.Network.HTTP.Request_Body.Request_Body
+import Standard.Base.Runtime.Managed_Resource.Managed_Resource
+
+import project.Connection.Snowflake_Details.Snowflake_Details
+
+polyglot java import org.enso.base.oauth.OAuthCallback
+
+perform_one_time_oauth account:Text role:Text =
+    uri = create_authorization_uri account role refresh_token=False
+    result = Managed_Resource.bracket (OAuthCallback.createCallbackServer 51234) (.close) server->
+        # We start the server first to ensure we can 'reserve' the port before opening the browser
+        IO.println "Please open "+uri.to_text+" in your browser and follow the instructions."
+        server.waitForCallback
+    code = OAuth_Callback_Response.decode result . code
+    exchange_one_time_code_for_access_token account code
+
+type OAuth_Callback_Response
+    Value code:Text state:Text|Nothing
+
+    decode response:Text -> OAuth_Callback_Response =
+        parts = response.split "&"
+        code_prefix = "code="
+        state_prefix = "state="
+        code = parts
+            . find (e-> e.starts_with code_prefix)
+            . drop code_prefix.length
+            . catch Not_Found _->
+                Error.throw (Illegal_State.Error "Malformed OAuth response: No code found.")
+        state = parts
+            . find (e-> e.starts_with state_prefix)
+            . drop state_prefix.length
+            . catch Not_Found _->Nothing
+        OAuth_Callback_Response.Value code state
+
+create_authorization_uri account:Text role:Text refresh_token:Boolean -> URI =
+    # TODO add code_challenge for PKCE
+    # TODO check that account does not contain any unexpected characters
+    base_uri = URI.from "https://"+account+".snowflakecomputing.com/oauth/authorize"
+    scope = (if refresh_token then "refresh_token " else "")+"session:role:"+role
+    base_uri
+        . add_query_argument "response_type" "code"
+        . add_query_argument "client_id" client_id
+        . add_query_argument "redirect_uri" redirect_uri
+        . add_query_argument "scope" scope
+        . add_query_argument "state" "foobar"
+
+type Access_Token
+    Value username:Text token:Text expiry:Date_Time
+
+    to_text self -> Text =
+        "Access_Token.Value "+self.username.pretty+" *** "+self.expiry.to_text
+
+    to_display_text self -> Text =
+        "Snowflake Access Token for "+self.username+" (expires at "+self.expiry.to_display_text+")"
+
+exchange_one_time_code_for_access_token account:Text code:Text -> Access_Token =
+    now = Date_Time.now
+    uri = create_token_uri account
+    params = Dictionary.from_vector [["grant_type", "authorization_code"], ["code", code], ["redirect_uri", redirect_uri]]
+    request_body = Request_Body.Form_Data params url_encoded=True
+    headers = [Header.authorization_basic client_id client_secret]
+    response = Data.post uri body=request_body headers=headers response_format=JSON_Format
+    Access_Token.Value (response.get "username") (response.get "access_token") (now + Duration.new seconds=(response.get "expires_in"))
+
+create_token_uri account:Text =
+    URI.from "https://"+account+".snowflakecomputing.com/oauth/token-request"
+
+authorize_for_refresh_token account:Text role:Text =
+    uri = create_authorization_uri account role refresh_token=True
+    result = Managed_Resource.bracket (OAuthCallback.createCallbackServer 51234) (.close) server->
+        # We start the server first to ensure we can 'reserve' the port before opening the browser
+        IO.println "Please open "+uri.to_text+" in your browser and follow the instructions."
+        server.waitForCallback
+    code = OAuth_Callback_Response.decode result . code
+    exchange_one_time_code_for_refresh_token account code
+
+exchange_one_time_code_for_refresh_token account:Text code:Text -> Refresh_Token =
+    now = Date_Time.now
+    uri = create_token_uri account
+    params = Dictionary.from_vector [["grant_type", "authorization_code"], ["code", code], ["redirect_uri", redirect_uri]]
+    request_body = Request_Body.Form_Data params url_encoded=True
+    headers = [Header.authorization_basic client_id client_secret]
+    response = Data.post uri body=request_body headers=headers response_format=JSON_Format
+    Refresh_Token.Value account (response.get "username") (response.get "refresh_token") (now + Duration.new seconds=(response.get "refresh_token_expires_in"))
+
+type Refresh_Token
+    Value account:Text username:Text token:Text expiry:Date_Time
+
+    to_text self -> Text =
+        "Refresh_Token.Value "+self.account.pretty+" "+self.username.pretty+" *** "+self.expiry.to_text
+
+    to_display_text self -> Text =
+        "Snowflake Token for "+self.account+"/"+self.username+" (expires at "+self.expiry.to_display_text+")"
+
+    is_expired self -> Boolean =
+        Date_Time.now >= self.expiry
+
+    check self -> Nothing ! Illegal_State =
+        if self.is_expired then
+            Error.throw (Illegal_State.Error "Refresh token has expired on "+self.expiry.to_display_text)
+
+    exchange_for_access_token self -> Access_Token ! Illegal_State =
+        self.check
+        now = Date_Time.now
+        uri = create_token_uri self.account
+        params = Dictionary.from_vector [["grant_type", "refresh_token"], ["refresh_token", self.token]]
+        request_body = Request_Body.Form_Data params url_encoded=True
+        headers = [Header.authorization_basic client_id client_secret]
+        response = Data.post uri body=request_body headers=headers response_format=JSON_Format
+        Access_Token.Value self.username (response.get "access_token") (now + Duration.new seconds=(response.get "expires_in"))
+
+client_id = Environment.get "SNOWFLAKE_APP_CLIENT_ID"
+client_secret = Environment.get "SNOWFLAKE_APP_CLIENT_SECRET"
+redirect_uri = "http://localhost:51234/oauth"

std-bits/base/src/main/java/org/enso/base/enso_cloud/ExternalLibrarySecretHelper.java

@@ -50,5 +50,8 @@ private static void checkAccess() throws EnsoSecretAccessDenied {
   private record AccessLocation(String className, String method) {}
 
   private static final List<AccessLocation> allowedAccessLocations =
-      List.of(new AccessLocation("org.enso.aws.ClientBuilder", "unsafeResolveSecrets"));
+      List.of(
+          new AccessLocation("org.enso.aws.ClientBuilder", "unsafeResolveSecrets"),
+          new AccessLocation("org.enso.google.GoogleOAuthSecretReader", "createCredentialFromSecretValue")
+      );
 }