diff --git a/guardrails_utilities/python_utils/delete_resources_batches/README.md b/guardrails_utilities/python_utils/delete_resources_batches/README.md new file mode 100644 index 00000000..c0ee0808 --- /dev/null +++ b/guardrails_utilities/python_utils/delete_resources_batches/README.md @@ -0,0 +1,184 @@ +# Delete Resources in batches from Guardrails + +This script first disables discovery-related policies for targeted resources that are planned for removal from Guardrails. This prevents discovery controls from re-running and re-adding the resources to the Guardrails [CMDB](https://turbot.com/guardrails/docs/reference/glossary#cmdb). + +When executed with the `--disable` flag, the script disables discovery policies for the specified resources and then deletes them **`along with their descendants`**. + +Deletion is performed in batches, with an optional `cooldown` period between batches to prevent system overload. + +For more details, refer to [Filtering Resources](https://turbot.com/guardrails/docs/reference/filter/resources). + +## Prerequisites + +To run the scripts, you must have: + +- [Python 3.\*.\*](https://www.python.org/downloads/) +- [Pip](https://pip.pypa.io/en/stable/installing/) + +## Setup + +This sections details how to set up an environment in order to run the script. + +### Virtual Environments Activation + +We recommend the use of [virtual environment](https://docs.python.org/3/library/venv.html). + +To setup a virtual environment: + +```shell +python3 -m venv .venv +``` + +Once created, to activate the environment: + +```shell +source .venv/bin/activate +``` + +### Dependencies + +Then install Python library dependencies: + +```shell +pip3 install -r requirements.txt +``` + +### Turbot Configuration + +Credentials and end point details need to be configure before being able to connect to a Turbot installation. +This configuration can be entered either using environment variables or a configuration file. + +#### Environment Variables + +Use either configuration for your Turbot installation: + +```shell +export TURBOT_WORKSPACE="https://.cloud.turbot.com/" +export TURBOT_ACCESS_KEY_ID=ac61d2e4-730c-4b54-8c3c-6ef172390814 +export TURBOT_SECRET_ACCESS_KEY=151b296b-0694-4a28-94c4-4b67fa82ab2c +``` + +or + +```shell +export TURBOT_GRAPHQL_ENDPOINT="https://.cloud.turbot.com/api/latest/graphql" +export TURBOT_ACCESS_KEY_ID=ac61d2e4-730c-4b54-8c3c-6ef180150814 +export TURBOT_SECRET_ACCESS_KEY=151b296b-0694-4a28-94c4-4767fa82bb2c +``` + +#### Configuration File + +Example configuration file: + +```yaml +default: + accessKey: dc61d2e4-730c-4b54-8c3c-6ef180150814 + secretKey: 6ef18015-7d0c-2b51-4d2c-dc61d2e63a22 + workspace: "https://demo-acme.cloud.turbot.com/" +``` + +This script will automatically search for a `credentials.yml` file in `~/.config/turbot/` or you can save the yaml configuration file anywhere and provide the `--config /path/to/config.yml --profile default` as a command line option. + +## Executing the Script + +To run a the Python script: + +1. Install and configure the [pre-requisites](#Prerequisites) +1. Using the command line, navigate to the directory for the Python script +1. Create and activate the Python virtual environment +1. Install dependencies +1. Run the Python script using the command line +1. Deactivate the Python virtual environment + +### Synopsis + +```shell +python3 delete_resources_batches.py [options] +``` + +### Options + +#### Details + +-c, --config-file + +> [String] Pass an optional yaml config file. + +-p, --profile + +> [String] Profile to be used from config file. + +-r, –resource-id + +> [String] ID of the resource to delete along with its descendants. + +-b, --batch + +> [Int] The number of controls to run before cooldown per cycle + +-d, --cooldown + +> [Int] Number of seconds to pause before the next batch of controls are run. Setting this value to `0` will disable cooldown. + +-e, --execute + +> Will re-run controls when found. + +–disable + +> Disable discovery-related policies by setting them to Skip before deletion. + +--help + +> Lists all the options and their usages. + +#### Example Usage + +##### Example 1: Simulate Deletion + +Simulates the deletion process without making any changes. + +```shell +python3 delete_resources_batches.py --resource-id +``` + +##### Example 2: Delete Resources and Disable Policies + +Disables discovery-related policies for the specified resource and deletes the resource along with its descendants. + +```shell +python3 delete_resources_batches.py --resource-id --disable --execute +``` + +##### Example 3: Delete Resources in Batches of 50 with a Cooldown + +Deletes resources in batches of 50, pausing for 60 seconds between batches. + +```shell +python3 delete_resources_batches.py --resource-id -b 50 -d 60 --execute +``` + +##### Example 4: Use a Configuration File for Credentials + +Runs the script using credentials specified in a configuration file. + +```shell +python3 delete_resources_batches.py --config ~/.config/turbot/credentials.yml --profile default --resource-id --execute +``` + +##### Example 5: Re-run controls in Multiple States + +```shell +python3 run_controls_batches.py -f "state:tbd,error,alarm" +``` + +## Virtual Environments Deactivation + +Once the script has been run, it is advised to deactivate the virtual environment if a virtual environment was used +to install the script dependencies. + +This is accomplished by running the command: + +```shell +deactivate +``` diff --git a/guardrails_utilities/python_utils/delete_resources_batches/delete_resources_batches.py b/guardrails_utilities/python_utils/delete_resources_batches/delete_resources_batches.py new file mode 100644 index 00000000..7f907de0 --- /dev/null +++ b/guardrails_utilities/python_utils/delete_resources_batches/delete_resources_batches.py @@ -0,0 +1,251 @@ +import turbot +import click +import requests +import sys +import time + + +@click.command() +@click.option('-c', '--config-file', type=click.Path(dir_okay=False), help="[String] Pass an optional YAML config file.") +@click.option('-p', '--profile', default="default", help="[String] Profile to be used from the config file.") +@click.option('-r', '--resource-id', required=True, help="[String] ID of the parent resource to delete along with its descendants.") +@click.option('-b', '--batch', default=100, help="[Int] Number of resources to delete per batch.") +@click.option('-d', '--cooldown', default=30, help="[Int] Cooldown time in seconds between batches.") +@click.option('--execute', is_flag=True, help="Execute the deletion process.") +@click.option('--disable', is_flag=True, help="Set discovery-related policies to 'Skip' before deletion.") +@click.option('-i', '--insecure', is_flag=True, help="Disable SSL certificate verification.") +def delete_resources(config_file, profile, resource_id, batch, cooldown, execute, disable, insecure): + """ + This script optionally disables discovery-related policies for a resource (--disable) + and then deletes the resource and its descendants. + """ + start_time = time.time() + + # Validate profile and connection + try: + config = turbot.Config(config_file, profile) + headers = {'Authorization': f'Basic {config.auth_token}'} + endpoint = config.graphql_endpoint + except Exception as e: + print(f"Error setting up configuration or connection: {e}") + sys.exit(1) + + # Handle SSL verification + requests.packages.urllib3.disable_warnings() + session = requests.Session() + session.verify = not insecure + + if disable: + print(f"\nDisabling discovery policies for resource ID: {resource_id}...") + handle_cmdb_policies(endpoint, headers, resource_id) + + print("\nFetching descendants for deletion...") + descendants = fetch_descendants(endpoint, headers, resource_id) + if not descendants: + print("No resources found to delete.") + return + + print(f"\nFound {len(descendants)} resources (including parent) to delete.") + + if not execute: + print("\n--execute flag not set. Exiting without deletion.") + return + + print("\nStarting resource deletion...") + delete_resources_in_batches(endpoint, headers, descendants, batch, cooldown) + + end_time = time.time() + elapsed_time = end_time - start_time + print(f"\nProcess completed in {elapsed_time:.2f} seconds.") + + +def handle_cmdb_policies(endpoint, headers, parent_resource_id): + """ + Fetch and set CMDB policies to 'Skip' for the given resource. + """ + cmdb_policy_query = ''' + query Policies($filter: [String!]!) { + policies: policyTypes(filter: $filter) { + items { + uri + } + metadata { + stats { + total + } + } + } + } + ''' + cmdb_policy_filter = f"resourceId:{parent_resource_id} Cmdb -CmdbAttributes" + policies = run_query(endpoint, headers, cmdb_policy_query, {'filter': cmdb_policy_filter})['data']['policies']['items'] + + if not policies: + print("No CMDB policies found.") + return + + print(f"Found {len(policies)} policies to update.") + + find_setting_query = ''' + query FindSetting($filter: [String!]!) { + policySettings(filter: $filter) { + items { + turbot { + id + } + } + metadata { + stats { + total + } + } + } + } + ''' + + for policy in policies: + policy_uri = policy['uri'] + # Create the filter dynamically + filter_value = [f"resourceId:'{parent_resource_id}' policyTypeId:'{policy_uri}'"] + # Pass the filter as a list + setting_result = run_query(endpoint, headers, find_setting_query, {'filter': filter_value}) + setting_exists = setting_result['data']['policySettings']['metadata']['stats']['total'] > 0 + + if setting_exists: + policy_setting_id = setting_result['data']['policySettings']['items'][0]['turbot']['id'] + update_mutation = ''' + mutation UpdatePolicySetting($input: UpdatePolicySettingInput!) { + updatePolicySetting(input: $input) { + turbot { + id + } + } + } + ''' + update_vars = { + "input": { + "id": policy_setting_id, + "precedence": "REQUIRED", + "value": "Skip", + "note": "Set via delete script to prevent re-discovery" + } + } + print(f"Updating policy {policy_uri} to 'Skip'...") + run_query(endpoint, headers, update_mutation, update_vars) + else: + create_mutation = ''' + mutation CreatePolicySetting($input: CreatePolicySettingInput!) { + createPolicySetting(input: $input) { + turbot { + id + } + } + } + ''' + create_vars = { + "input": { + "type": policy_uri, + "resource": parent_resource_id, + "precedence": "REQUIRED", + "value": "Skip", + "note": "Set via delete script to prevent re-discovery" + } + } + print(f"Creating policy {policy_uri} with 'Skip'...") + run_query(endpoint, headers, create_mutation, create_vars) + + +def fetch_descendants(endpoint, headers, parent_resource_id): + """ + Fetch descendants of a resource for deletion. + """ + query = ''' + query Descendants($id: ID!, $filter: [String!], $paging: String) { + resource(id: $id) { + children(filter: $filter, paging: $paging) { + items { + turbot { + id + } + } + paging { + next + } + } + } + } + ''' + variables = {'id': parent_resource_id, 'filter': ["level:descendant"], 'paging': None} + result = run_query(endpoint, headers, query, variables) + + # Check if the resource exists + if not result.get('data') or not result['data'].get('resource'): + print(f"Resource with ID {parent_resource_id} does not exist.") + return [] + + # Proceed to fetch descendants + descendants = [] + while True: + children = result['data']['resource']['children'] + items = children.get('items', []) + descendants.extend(items) + + paging = children.get('paging', {}).get('next') + if not paging: + break + + # Fetch next page of descendants + variables['paging'] = paging + result = run_query(endpoint, headers, query, variables) + + # Add the parent resource itself for deletion + descendants.append({'turbot': {'id': parent_resource_id}}) + return descendants + +def delete_resources_in_batches(endpoint, headers, resources, batch_size, cooldown): + """ + Delete resources in batches with cooldown periods. + """ + mutation = ''' + mutation DeleteResource($input: DeleteResourceInput!) { + deleteResource(input: $input) { + turbot { + id + } + } + } + ''' + total_resources = len(resources) + completed = 0 + + for index, resource in enumerate(resources): + resource_id = resource['turbot']['id'] + variables = {'input': {'id': resource_id}} + print(f"Deleting resource {resource_id}...") + run_query(endpoint, headers, mutation, variables) + completed += 1 + + # Cooldown logic + if completed % batch_size == 0 and completed < total_resources: + print(f"Cooldown: Waiting {cooldown} seconds before the next batch...") + time.sleep(cooldown) + + print(f"Deleted {completed} resources.") + + +def run_query(endpoint, headers, query, variables): + """ + Helper function to execute a GraphQL query or mutation. + """ + response = requests.post(endpoint, headers=headers, json={'query': query, 'variables': variables}, verify=False) + if response.status_code == 200: + return response.json() + else: + raise Exception(f"Query failed with status code {response.status_code}: {response.text}") + + +if __name__ == "__main__": + if sys.version_info >= (3, 5): + delete_resources() + else: + print("This script requires Python 3.5 or newer.") \ No newline at end of file diff --git a/guardrails_utilities/python_utils/delete_resources_batches/requirements.txt b/guardrails_utilities/python_utils/delete_resources_batches/requirements.txt new file mode 100644 index 00000000..59bdec45 --- /dev/null +++ b/guardrails_utilities/python_utils/delete_resources_batches/requirements.txt @@ -0,0 +1,3 @@ +Click>=8.1.6 +sgqlc>=16.3 +../turbot \ No newline at end of file