diff --git a/guardrails_utilities/python_utils/gcp_project_control_usage_summary/README.md b/guardrails_utilities/python_utils/gcp_project_control_usage_summary/README.md new file mode 100644 index 00000000..a0b29fa2 --- /dev/null +++ b/guardrails_utilities/python_utils/gcp_project_control_usage_summary/README.md @@ -0,0 +1,189 @@ +# Run GCP Project Controls + +Retrieves all GCP projects and runs a subquery on each to get control counts and other details. + +For further reference, see [Turbot's GraphQL API documentation](https://turbot.com/guardrails/docs/reference/graphql#graphql). + +## Prerequisites + +To run the script, you must have: + +- [Python 3.\*.\*](https://www.python.org/downloads/) +- [Pip](https://pip.pypa.io/en/stable/installing/) + +## Setup + +This section details how to set up an environment in order to run the script. + +### Virtual Environment Activation + +We recommend the use of a [virtual environment](https://docs.python.org/3/library/venv.html). + +To set up a virtual environment: + +```shell +python3 -m venv .venv +``` + +Once created, to activate the environment: + +```shell +source .venv/bin/activate +``` + +### Dependencies + +Then install Python library dependencies: + +```shell +pip3 install -r requirements.txt +``` + +### Turbot configuration + +Credentials and end point details need to be configure before being able to connect to a Turbot installation. +This configuration can be entered either using environment variables or a configuration file. + +#### Environment variables + +Use either configuration for your Turbot installation: + +```shell +export TURBOT_WORKSPACE="https://.cloud.turbot.com/" +export TURBOT_ACCESS_KEY_ID=ac61d2e4-730c-4b54-8c3c-6ef172390814 +export TURBOT_SECRET_ACCESS_KEY=151b296b-0694-4a28-94c4-4b67fa82ab2c +``` + +or + +```shell +export TURBOT_GRAPHQL_ENDPOINT="https://.cloud.turbot.com/api/latest/graphql" +export TURBOT_ACCESS_KEY_ID=ac61d2e4-730c-4b54-8c3c-6ef180150814 +export TURBOT_SECRET_ACCESS_KEY=151b296b-0694-4a28-94c4-4767fa82bb2c +``` + +#### Configuration file + +Example configuration file: + +```yaml +default: + accessKey: dc61d2e4-730c-4b54-8c3c-6ef180150814 + secretKey: 6ef18015-7d0c-2b51-4d2c-dc61d2e63a22 + workspace: "https://demo-acme.cloud.turbot.com/" +``` + +This script will automatically search for a `credentials.yml` file in `~/.config/turbot/` or you can save the yaml configuration file anywhere and provide the `--config /path/to/config.yml --profile default` as a command line option. + +## Executing the script + +To run a the Python script: + +1. Install and configure the [prerequisites](#prerequisites) +1. Using the command line, navigate to the directory for the Python script +1. Create and activate the Python virtual environment +1. Install dependencies +1. Run the Python script using the command line +1. Deactivate the Python virtual environment + +### Synopsis + +```shell +python3 gcp_project_control_usage_summary.py [options] +``` + +### Options + +#### Details + +-c, --config-file + +> (Optional) [String] Path to a YAML configuration file. If not provided, the script will look for credentials in the default location (~/.config/turbot/credentials.yml). + +-p, --profile + +> (Optional) [String] Profile name to be used from the configuration file. Defaults to `default` if not specified. + +-i, --insecure + +> (Optional) [Flag] Disables SSL certificate verification. Use this only if connecting to environments with self-signed certificates. + +--project-turbot-id + +> (Optional) [String] Turbot ID of a specific GCP project to fetch usage for. If not provided, the script fetches usage for all projects within the workspace. + +--all-control-states + +> (Optional) [Flag] Includes all control states (active, inactive, etc.). If not provided, the script fetches usage for active controls only (default behavior). + +--help + +> Displays all available options with descriptions and usage examples. + +#### Example usage + +##### Example 1 + +Run the script using default credentials + +```shell +python3 gcp_project_control_usage_summary.py --project-turbot-id 200001000010000 +``` + +##### Example 2 + +Run the script using a specific credentials file credentials.yml + +```shell +python3 gcp_project_control_usage_summary.py -c /path/to/credentials.yml -p profile1 --project-turbot-id 200001000010000 +``` + +##### Example 3 + +Run the script using a specific Turbot CLI profile + +```shell +python3 gcp_project_control_usage_summary.py -p demo-dev --project-turbot-id 200001000010000 +``` + +##### Example 4 + +Run the script for a specific GCP Project with Turbot ID 200001000010000 + +```shell +python3 gcp_project_control_usage_summary.py -p demo-dev --project-turbot-id 200001000010000 +``` + +##### Example 5 + +Run the script for all GCP Projects using a specific Turbot CLI profile + +```shell +python3 gcp_project_control_usage_summary.py -p demo-dev +``` + +##### Example 6 + +Run the script to fetch the usage (active controls only) for all the workspaces within the workspace + +```shell +python3 gcp_project_control_usage_summary.py -p demo-dev +``` + +##### Example 6 + +Run the script to fetch the usage (active and inactive controls) for all the workspaces within the workspace + +```shell +python3 gcp_project_control_usage_summary.py -p demo-dev --all-control-states +``` + +## Virtual environments deactivation + +Once the script has been run, it is advised to deactivate the virtual environment if a virtual environment was used to install the script dependencies. + +This is accomplished by running the command: + +```shell +deactivate +``` diff --git a/guardrails_utilities/python_utils/gcp_project_control_usage_summary/gcp_project_control_usage_summary.py b/guardrails_utilities/python_utils/gcp_project_control_usage_summary/gcp_project_control_usage_summary.py new file mode 100644 index 00000000..a0f5d357 --- /dev/null +++ b/guardrails_utilities/python_utils/gcp_project_control_usage_summary/gcp_project_control_usage_summary.py @@ -0,0 +1,210 @@ +import click +import turbot +import requests +from datetime import datetime + +# Counter for API calls +api_call_count = 0 + +def fetch_projects(session, api_url, headers, project_turbot_id): + global api_call_count + query = ''' + query ($filter: [String!], $paging: String) { + resources(filter: $filter, paging: $paging) { + items { + trunk { + title + } + turbot { + title + id + } + } + paging { + next + } + } + } + ''' + projects = [] + paging = None + + if project_turbot_id: + filter_str = f"resourceId:{project_turbot_id} resourceTypeId:tmod:@turbot/gcp#/resource/types/project level:self limit:5000" + else: + filter_str = "resourceTypeId:tmod:@turbot/gcp#/resource/types/project level:self limit:5000" + + while True: + variables = {'filter': [filter_str], 'paging': paging} # Wrap filter in a list + response = session.post( + api_url, + json={'query': query, 'variables': variables}, + headers=headers + ) + api_call_count += 1 # Increment API call count + + if not response.ok: + print(f"Error fetching projects: {response.text}") + response.raise_for_status() + + data = response.json() + + if "errors" in data: + for error in data["errors"]: + print(f"Error: {error['message']}") + break + + projects.extend(data['data']['resources']['items']) + paging = data['data']['resources']['paging']['next'] + if not paging: + break + + return projects + +def fetch_control_data(session, api_url, headers, resource_id, control_type_id, all_control_states): + global api_call_count + query = ''' + query ($filter: [String!], $paging: String) { + controlSummariesByControlType(filter: $filter, paging: $paging) { + items { + summary { + control { + alarm + error + invalid + muted + ok + skipped + tbd + total + } + } + type { + uri + trunk { + title + } + } + } + paging { + next + } + } + } + ''' + controls = [] + paging = None + + if all_control_states: + filter_str = f"resourceId:{resource_id} controlTypeId:{control_type_id} limit:5000" + else: + filter_str = f"resourceId:{resource_id} controlTypeId:{control_type_id} state:active limit:5000" + + while True: + variables = {'filter': [filter_str], 'paging': paging} # Wrap filter in a list + response = session.post( + api_url, + json={'query': query, 'variables': variables}, + headers=headers + ) + api_call_count += 1 # Increment API call count + + if not response.ok: + print(f"Error fetching control summaries: {response.text}") + response.raise_for_status() + + data = response.json() + + if "errors" in data: + for error in data["errors"]: + print(f"Error: {error['message']}") + break + + controls.extend(data['data']['controlSummariesByControlType']['items']) + paging = data['data']['controlSummariesByControlType']['paging']['next'] + if not paging: + break + + return controls + +def process_control_data(session, api_url, headers, resource_id, control_type_id, project_title, trunk_title, output_lines, all_control_states): + items = fetch_control_data(session, api_url, headers, resource_id, control_type_id, all_control_states) + current_date = datetime.now().strftime("%m-%d-%Y") + for item in items: + control_summary = item['summary']['control'] + type_uri = item['type']['uri'] + control_title = item['type']['trunk']['title'] + + # Only include control/types in the output + if 'control/types' in type_uri: + output_lines.append( + f"{current_date},{project_title},{trunk_title},{control_title},{type_uri},{control_summary['alarm']},{control_summary['error']},{control_summary['invalid']},{control_summary['muted']},{control_summary['ok']},{control_summary['skipped']},{control_summary['tbd']},{control_summary['total']}" + ) + + if 'resource/types' in type_uri: + process_control_data(session, api_url, headers, resource_id, type_uri, project_title, trunk_title, output_lines, all_control_states) + +@click.command() +@click.option('-c', '--config-file', type=click.Path(dir_okay=False), help="[String] Pass an optional yaml config file.") +@click.option('-p', '--profile', default="default", help="[String] Profile to be used from config file.") +@click.option('-i', '--insecure', is_flag=True, help="Disable SSL certificate verification.") +@click.option('--project-turbot-id', default=None, help="[String] Turbot ID of a single project to fetch usage for. If not provided, fetches usage for all projects.") +@click.option('--all-control-states', is_flag=True, help="Include all control states instead of only active ones.") +def get_workspace_usage(config_file, profile, insecure, project_turbot_id, all_control_states): + global api_call_count + start_time = datetime.now() + print(f"Script started at {start_time}\n") + + try: + config = turbot.Config(config_file, profile) + headers = {'Authorization': f'Basic {config.auth_token}'} + api_url = config.graphql_endpoint # Use the endpoint directly from config + except Exception as e: + print(f"Error: Unable to load configuration or connect to the endpoint. {e}") + return + + session = requests.Session() + session.verify = not insecure + + if insecure: + requests.packages.urllib3.disable_warnings() + + if project_turbot_id: + if all_control_states: + print(f"Collecting the usage summary (active and inactive) for the project with Turbot ID: {project_turbot_id}...\n") + else: + print(f"Collecting the active usage summary for the project with Turbot ID: {project_turbot_id}...\n") + else: + if all_control_states: + print("Collecting the usage summary (active and inactive) for all projects...\n") + else: + print("Collecting the active usage summary for all projects...\n") + + api_call_count = 0 # Initialize API call count before making any requests + projects = fetch_projects(session, api_url, headers, project_turbot_id) + output_lines = [] + current_date = datetime.now().strftime("%m-%d-%Y") + output_file = f'{profile}_control_usage_summary_{datetime.now().strftime("%m%d%Y")}.csv' # Dynamic file name with date + + for project in projects: + project_id = project['turbot']['id'] + project_title = project['turbot']['title'] + trunk_title = project['trunk']['title'] + + process_control_data(session, api_url, headers, project_id, "tmod:@turbot/gcp#/resource/types/gcp", project_title, trunk_title, output_lines, all_control_states) + + with open(output_file, 'w') as f: + f.write('Date,Project Title,Trunk Title,Control Title,Type URI,Alarm,Error,Invalid,Muted,OK,Skipped,TBD,Total\n') + for line in output_lines: + f.write(f"{line}\n") + + end_time = datetime.now() + elapsed_time = (end_time - start_time).total_seconds() + + print(f"Data successfully written to {output_file}") + print(f"Script completed at {end_time}") + print(f"Total time taken: {elapsed_time:.2f} seconds") + print(f"Total API calls made: {api_call_count}") + +if __name__ == '__main__': + get_workspace_usage() diff --git a/guardrails_utilities/python_utils/gcp_project_control_usage_summary/requirements.txt b/guardrails_utilities/python_utils/gcp_project_control_usage_summary/requirements.txt new file mode 100644 index 00000000..8e25c374 --- /dev/null +++ b/guardrails_utilities/python_utils/gcp_project_control_usage_summary/requirements.txt @@ -0,0 +1,7 @@ +Click>=8.1.6 +sgqlc>=16.3 +PyYAML>=6.0.1 +requests>=2.31.0 +xdg>=6.0.0 +urllib3==1.25.10 +../turbot \ No newline at end of file