[DAPS-1731] - Feature, scripts, compose - add scripts to generate globus credentials for web service (#1731)

Co-authored-by: Joshua S Brown <[email protected]>
Co-authored-by: sourcery-ai[bot] <58596630+sourcery-ai[bot]@users.noreply.github.com>

Author: 3 people

compose/all/generate_globus_files.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
-SCRIPT=$(realpath "$0")
+SCRIPT=$(realpath "${BASH_SOURCE[0]}")
 SOURCE=$(dirname "$SCRIPT")
 PROJECT_ROOT=$(realpath "${SOURCE}/../../")
 
+set -euf -o pipefail
+
 "${PROJECT_ROOT}/scripts/compose_generate_globus_files.sh" -d "$(pwd)"

compose/metadata/generate_globus_files.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+SCRIPT=$(realpath "${BASH_SOURCE[0]}")
+SOURCE=$(dirname "$SCRIPT")
+PROJECT_ROOT=$(realpath "${SOURCE}/../../")
+
+set -euf -o pipefail
+
+"${PROJECT_ROOT}/scripts/compose_generate_web_server_globus_credentials.sh" -d "$(pwd)"

compose/repo/generate_globus_files.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
-SCRIPT=$(realpath "$0")
+SCRIPT=$(realpath "${BASH_SOURCE[0]}")
 SOURCE=$(dirname "$SCRIPT")
 PROJECT_ROOT=$(realpath "${SOURCE}/../../")
 
+set -euf -o pipefail
+
 "${PROJECT_ROOT}/scripts/compose_generate_globus_files.sh" -d "$(pwd)"

scripts/compose_generate_globus_files.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-SCRIPT=$(realpath "$0")
+SCRIPT=$(realpath "${BASH_SOURCE[0]}")
 SOURCE=$(dirname "$SCRIPT")
 PROJECT_ROOT=$(realpath "${SOURCE}/..")
 

scripts/compose_generate_web_server_globus_credentials.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+SCRIPT=$(realpath "${BASH_SOURCE[0]}")
+SOURCE=$(dirname "$SCRIPT")
+PROJECT_ROOT=$(realpath "${SOURCE}/..")
+
+source "${PROJECT_ROOT}/scripts/dependency_versions.sh"
+
+# This script should be run after generating the .env file as it will pull
+# values from the .env file
+Help() {
+  echo "$(basename $0) generate credentials for the DataFed Web Server."
+  echo "Note the .env file must exist in the specified directory."
+  echo
+  echo "Syntax: $(basename $0) [-h|d]"
+  echo "options:"
+  echo "-h, --help                        Print this help message"
+  echo "-d, --directory                   Directory where globus folder will be"
+  echo "                                  created."
+}
+
+VALID_ARGS=$(getopt -o hd: --long 'help',directory: -- "$@")
+
+DIRECTORY=""
+eval set -- "$VALID_ARGS"
+while [ : ]; do
+  case "$1" in
+  -h | --help)
+    Help
+    exit 0
+    ;;
+  -d | --directory)
+    DIRECTORY="$2"
+    shift 2
+    ;;
+  --)
+    shift
+    break
+    ;;
+  \?) # incorrect option
+    echo "Error: Invalid option"
+    exit
+    ;;
+  esac
+done
+
+if [ ! -d "${DIRECTORY}" ]; then
+  echo "The provided directory does not seem to exist: ${DIRECTORY}"
+fi
+
+if [ ! -f "${DIRECTORY}/.env" ]; then
+  echo "Missing . ${DIRECTORY}/.env file. This file needs to be"
+  echo "created first"
+  exit 1
+fi
+
+# This script should be run after generating the .env file as it will pull
+# values from the .env file
+
+if [ ! -f "${DIRECTORY}/.env" ]; then
+  echo "Missing . ${DIRECTORY}/.env file. This file needs to be"
+  echo "created first"
+  exit 1
+fi
+
+local_DATAFED_GLOBUS_KEY_DIR="${DIRECTORY}/globus"
+if [ ! -d "$local_DATAFED_GLOBUS_KEY_DIR" ]; then
+  mkdir -p "$local_DATAFED_GLOBUS_KEY_DIR"
+fi
+
+# Because docker compose honors spaces and reads in .env files as literals
+# we cannot include the quotes for variables that have spaces. So we need to
+# convert this file such that it is in a format that can be readable by bash
+# before loading it into the env
+
+cp "${DIRECTORY}/.env" "${DIRECTORY}/.env_shell"
+
+sed -i 's/=\([^"]*\)/="\1"/' "${DIRECTORY}/.env_shell"
+
+. "${DIRECTORY}/.env_shell"
+
+# Cleanup after loading env
+rm "${DIRECTORY}/.env_shell"
+
+DATAFED_GLOBUS_REDIRECT_CRED_FILE_PATH="$DATAFED_GLOBUS_REDIRECT_CRED_FILE_PATH" \
+  DATAFED_GCS_ROOT_NAME="$DATAFED_GCS_ROOT_NAME" \
+  DATAFED_DOMAIN="$DATAFED_DOMAIN" \
+  "python${DATAFED_PYTHON_VERSION}" "${PROJECT_ROOT}/scripts/globus/generate_web_server_credentials.py"

scripts/globus/generate_web_server_credentials.py

@@ -0,0 +1,107 @@
+import globus_sdk
+from globus_sdk.scopes import GroupsScopes
+from globus_sdk import AuthClient, GroupsClient
+
+import utils
+
+import os, sys
+
+CLIENT_ID = "f8d0afca-7ac4-4a3c-ac05-f94f5d9afce8"
+
+# The Globus project the GCS endpoint will be created in
+default_DATAFED_GCS_ROOT_NAME = "DataFed Repo"
+DATAFED_GCS_ROOT_NAME = os.getenv(
+    "DATAFED_GCS_ROOT_NAME", default_DATAFED_GCS_ROOT_NAME
+)
+if len(DATAFED_GCS_ROOT_NAME) == 0:
+    CRED_FILE_PATH = default_DATAFED_GCS_ROOT_NAME
+
+default_PROJECT_NAME = f"{DATAFED_GCS_ROOT_NAME} Project"
+PROJECT_NAME = os.getenv("DATAFED_GLOBUS_PROJECT_NAME", default_PROJECT_NAME)
+if len(PROJECT_NAME) == 0:
+    PROJECT_NAME = default_PROJECT_NAME
+
+default_REDIRECT_CLIENT_NAME = DATAFED_GCS_ROOT_NAME + " Web Server Client"
+REDIRECT_CLIENT_NAME = os.getenv("DATAFED_GLOBUS_REDIRECT_CLIENT_NAME", default_REDIRECT_CLIENT_NAME)
+if len(REDIRECT_CLIENT_NAME) == 0:
+    REDIRECT_CLIENT_NAME = default_REDIRECT_CLIENT_NAME
+
+default_REDIRECT_CRED_NAME = DATAFED_GCS_ROOT_NAME + " Web Server Cred"
+REDIRECT_CRED_NAME = os.getenv("DATAFED_GLOBUS_REDIRECT_CRED_NAME", default_REDIRECT_CRED_NAME)
+if len(REDIRECT_CRED_NAME) == 0 :
+    REDIRECT_CRED_NAME = default_REDIRECT_CRED_NAME
+
+default_REDIRECT_CRED_FILE_PATH = os.path.abspath("./globus/globus_config.json")
+REDIRECT_CRED_FILE_PATH = os.getenv("DATAFED_GLOBUS_REDIRECT_CRED_FILE_PATH", default_REDIRECT_CRED_FILE_PATH)
+if len(REDIRECT_CRED_FILE_PATH) == 0:
+    REDIRECT_CRED_FILE_PATH = default_REDIRECT_CRED_FILE_PATH
+
+default_DOMAIN = "localhost"
+DOMAIN = os.getenv("DATAFED_DOMAIN", default_DOMAIN)
+if len(DOMAIN) == 0:
+    DOMAIN = default_DOMAIN
+REDIRECT_PATH = os.path.join("https://", DOMAIN, "ui/authn")
+
+# begin oauth
+client = globus_sdk.NativeAppAuthClient(CLIENT_ID)
+
+group_scope = GroupsScopes.make_mutable("all")
+client.oauth2_start_flow(
+    requested_scopes="openid profile email "
+    "urn:globus:auth:scope:auth.globus.org:manage_projects "
+    "urn:globus:auth:scope:auth.globus.org:view_identities " + str(group_scope),
+    refresh_tokens=True,
+)
+
+authorize_url = client.oauth2_get_authorize_url(query_params={"prompt": "login"})
+print("Please go to this URL and login: \n", authorize_url)
+auth_code = input("Please enter the authorization code: ")
+
+token_response = client.oauth2_exchange_code_for_tokens(auth_code)
+# Extract the token
+refresh_token_auth = token_response.by_resource_server["auth.globus.org"][
+    "refresh_token"
+]
+refresh_token_groups = token_response.by_resource_server["groups.api.globus.org"][
+    "refresh_token"
+]
+rt_authorizer = globus_sdk.RefreshTokenAuthorizer(refresh_token_auth, client)
+
+rt_authorizer_groups = globus_sdk.RefreshTokenAuthorizer(refresh_token_groups, client)
+
+# auth_client_refresh_token
+ac_rt = AuthClient(authorizer=rt_authorizer)
+gr_rt = GroupsClient(authorizer=rt_authorizer_groups)
+
+userinfo = ac_rt.oauth2_userinfo()
+# Will get the primary email and id
+identity_id = userinfo["sub"]
+email = userinfo["email"]
+username = userinfo["preferred_username"]
+print("username")
+print(username)
+print("userinfo")
+print(userinfo)
+organization = userinfo["identity_provider_display_name"]
+
+# Need to determine the project uuid
+if utils.projectExists(ac_rt, PROJECT_NAME) is False:
+    project_id = utils.createProject(ac_rt, PROJECT_NAME, userinfo)
+else:
+    project_id = utils.getProjectId(ac_rt, PROJECT_NAME)
+
+count = utils.countProjects(ac_rt, PROJECT_NAME)
+
+if count != 1:
+    print(
+        "Something is wrong there should be at least one project with name"
+        f" {PROJECT_NAME} instead there are {count} with that name"
+    )
+    sys.exit(1)
+
+
+print(f"Project id is {project_id}")
+
+redirect_c_id, redirect_c_secret = utils.createClient(
+    ac_rt, REDIRECT_CLIENT_NAME, project_id, REDIRECT_CRED_NAME, REDIRECT_CRED_FILE_PATH, REDIRECT_PATH
+)

scripts/globus/initialize_globus_endpoint.py

@@ -35,7 +35,7 @@
     CRED_NAME = default_CRED_NAME
 
 # Name of the file where we will store confidential client credentials
-default_CRED_FILE_PATH = os.path.abspath("./globus/client_cred.json")
+default_CRED_FILE_PATH = os.path.abspath("./globus/globus_config.json")
 CRED_FILE_PATH = os.getenv("DATAFED_GLOBUS_CRED_FILE_PATH", default_CRED_FILE_PATH)
 if len(CRED_FILE_PATH) == 0:
     CRED_FILE_PATH = default_CRED_FILE_PATH

scripts/globus/utils.py

@@ -131,6 +131,18 @@ def createNewClient(auth_client, client_name, project_id):
 
     return client_id
 
+def createNewRedirectClient(auth_client, client_name, project_id, redirect_uri):
+    client_id = getClientId(auth_client, client_name, project_id)
+
+    client_exists = bool(client_id)
+    if not client_exists:
+        result = auth_client.create_client(
+            client_name, project=project_id, public_client=False,
+            redirect_uris=[redirect_uri]
+        )
+        client_id = result["client"]["id"]
+
+    return client_id
 
 def getCredentialID(auth_client, client_id, cred_name):
     get_client_cred_result = auth_client.get_client_credentials(client_id)
@@ -179,29 +191,41 @@ def validFile(file_name):
     return file_exists, file_empty
 
 
-def getCredentialFromFile(cred_file_name, cred_id):
+def getCredentialFromFile(cred_file_name, cred_id, cred_type="setup"):
     # Check to see if the local secret is the same id and not just the same
     # name
     _, cred_empty = validFile(cred_file_name)
     if cred_empty is False:
         with open(cred_file_name, "r") as f:
             loaded_data = json.load(f)
+            if cred_type in loaded_data.keys():
+                loaded_data = loaded_data[cred_type]
+            else :
+                print(f"Failed to get credential from file '{cred_file_name}': credential type '{cred_type}' not found.")
             if loaded_data["client"] == cred_id:
                 return loaded_data["secret"]
     return None
 
 
-def getClientIdFromCredFile(cred_file_name):
+def getClientIdFromCredFile(cred_file_name, cred_type="setup"):
     # Check to see if the local secret is the same id and not just the same
     # name
     _, cred_empty = validFile(cred_file_name)
     if cred_empty is False:
         with open(cred_file_name, "r") as f:
             loaded_data = json.load(f)
-            return loaded_data["client"]
+            if cred_type in loaded_data.keys():
+                loaded_data = loaded_data[cred_type]
+                return loaded_data["client"]
+            else :
+                print(f"Failed to get client ID from file '{cred_file_name}': credential type '{cred_type}' not found.")
     return None
 
 
+# Doesn't appear to be used.
+# Not removing until we have a chance to refactor the entire
+#   globus configuration setup process.
+
 def getEndpointIdFromFile(deployment_key_file_path):
     # Check to see if the local secret is the same id and not just the same
     # name
@@ -213,22 +237,24 @@ def getEndpointIdFromFile(deployment_key_file_path):
     return None
 
 
-def createNewCredential(auth_client, client_id, cred_name, cred_file):
-
+def createNewCredential(auth_client, client_id, cred_name, cred_file, cred_type):
     get_client_cred_result = auth_client.get_client_credentials(client_id)
     for cred in get_client_cred_result["credentials"]:
         # Should have stored secret locally
         auth_client.delete_client_credential(client_id, cred["id"])
 
     cred_result = auth_client.create_client_credential(client_id, cred_name)
     # Have to change this to a dict
-    obj = {
-        "client": cred_result["credential"]["client"],
-        "id": cred_result["credential"]["id"],
-        "name": cred_result["credential"]["name"],
-        "secret": cred_result["credential"]["secret"],
-    }
 
+    obj = { cred_type: 
+        {
+            "client": cred_result["credential"]["client"],
+            "id": cred_result["credential"]["id"],
+            "name": cred_result["credential"]["name"],
+            "secret": cred_result["credential"]["secret"],
+        }
+    }
+    
     # Check that the folder exists
     folder_path = os.path.dirname(cred_file)
     if not os.path.exists(folder_path):
@@ -246,9 +272,9 @@ def createNewCredential(auth_client, client_id, cred_name, cred_file):
     return cred_result["credential"]["secret"]
 
 
-def getClientSecret(auth_client, client_id, cred_name, cred_id, cred_file):
+def getClientSecret(auth_client, client_id, cred_name, cred_id, cred_file, cred_type):
 
-    client_secret = getCredentialFromFile(cred_file, cred_id)
+    client_secret = getCredentialFromFile(cred_file, cred_id, cred_type)
 
     create_new_credential = True
     remove_cached_credential = True
@@ -268,23 +294,30 @@ def getClientSecret(auth_client, client_id, cred_name, cred_id, cred_file):
     if create_new_credential:
         # Remove credentials from cloud
         client_secret = createNewCredential(
-            auth_client, client_id, cred_name, cred_file
+            auth_client, client_id, cred_name, cred_file, cred_type
         )
 
     return client_secret
 
 
-def createClient(auth_client, client_name, project_id, cred_name, cred_file):
-    client_id = createNewClient(auth_client, client_name, project_id)
+def createClient(auth_client, client_name, project_id, cred_name, cred_file, redirect_uri=None):
+    if redirect_uri is not None:
+        client_id = createNewRedirectClient(auth_client, client_name, project_id, redirect_uri)
+        cred_type = "web"
+    else :
+        # if we haven't provided a redirect_uri, assume we're trying to
+        # create a setup client, because that was previously the only
+        # time this function was called.
+        client_id = createNewClient(auth_client, client_name, project_id)
+        cred_type = "setup"
 
     cred_id = getCredentialID(auth_client, client_id, cred_name)
 
     client_secret = getClientSecret(
-        auth_client, client_id, cred_name, cred_id, cred_file
+        auth_client, client_id, cred_name, cred_id, cred_file, cred_type
     )
     return client_id, client_secret
 
-
 def getGCSClientIDFromDeploymentFile(deployment_key_file):
     deployment_key_exists, deployment_key_empty = validFile(deployment_key_file)