From 0b6f82c23fdb833839e5dc9272d398790cca40a6 Mon Sep 17 00:00:00 2001 From: Kalibh Halford Date: Fri, 14 Feb 2025 11:35:34 +0000 Subject: [PATCH] MAINT: Remove weekly reporting from this repository Moving to cloud-ops-tools --- README.md | 4 - Weekly-Reporting/README.md | 31 ---- Weekly-Reporting/data.yaml.template | 57 ------- Weekly-Reporting/export.py | 232 ---------------------------- Weekly-Reporting/format_raw_data.py | 79 ---------- Weekly-Reporting/report.sh | 15 -- Weekly-Reporting/requirements.txt | 2 - 7 files changed, 420 deletions(-) delete mode 100644 Weekly-Reporting/README.md delete mode 100644 Weekly-Reporting/data.yaml.template delete mode 100644 Weekly-Reporting/export.py delete mode 100644 Weekly-Reporting/format_raw_data.py delete mode 100644 Weekly-Reporting/report.sh delete mode 100644 Weekly-Reporting/requirements.txt diff --git a/README.md b/README.md index bed31a4b..21975bac 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,3 @@ One terraform script to create a private network on openstack along with a route A Python script that when run, creates a filter word cloud from the summary of tickets over a time period. [More Here](word_cloud_generator/) - -# Weekly-Reporting -A Python script to upload data to the weekly reporting InfluxDB instance for the Cloud statistics reporting. -[More Here](Weekly-Reporting/) diff --git a/Weekly-Reporting/README.md b/Weekly-Reporting/README.md deleted file mode 100644 index dae4f9fc..00000000 --- a/Weekly-Reporting/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# Weekly Reporting - -This script is used to upload a weekly report to an InfluxDB instance. It tags all the data points with the time when ran.
-If you need to back-fill data or use a different time you can do the below in the code:
-``` -... -16 # time = datetime.datetime.now().isoformat() -17 time = "2025-01-02T15:17:37.780483" -... -``` -The script uses argparse to provide an easy-to-use command line interface.
-There is a template yaml file [here](data.yaml) which **requires all values to be not empty**. - -> **NOTE on data.yaml:** -> - Values in data.yaml must not be empty. -> - Values which can be floating point such as Memory in TB must have .0 for whole numbers. - -## Instructions: - -1. Source your openstack cli venv and OpenStack admin credentials. -2. Run the `report.sh` script to generate the data file. -2. Write your token in plain text in a file e.g. "token" -3. Run `export.py` with the correct arguments, see below: - -```shell -python3 export.py --host="http://172.16.103.52:8086" \ ---org="cloud" \ ---bucket="weekly-reports-time" ---token-file="token" ---report-file="data-9.yaml" -``` diff --git a/Weekly-Reporting/data.yaml.template b/Weekly-Reporting/data.yaml.template deleted file mode 100644 index 2b77098a..00000000 --- a/Weekly-Reporting/data.yaml.template +++ /dev/null @@ -1,57 +0,0 @@ -# This is the data file for the weekly statistics report exporter. -# -# Note the below. -# -# Every field must have a value. -# Where values are floating type, they must remain floating type. -# For example, if memory_in_use changes from 234.11 TB to 212 TB. -# You must enter 212 TB as 212.0 TB - - -cpu: - # Cores in the Cloud - in_use: - total: - -memory: - # Memory in the Cloud (TB) - in_use: - total: - -storage: - # Storage stats by type (TB) - sirius: - in_use: - total: - nodes: - deneb: - in_use: - total: - arided: - in_use: - total: - -hv: - # HV stats - active: - active: - cpu_full: - memory_full: - down: - disabled: - -vm: - # VM stats - active: - shutoff: - errored: - building: - -floating_ip: - # Floating IP - in_use: - total: - -virtual_worker_nodes: - # Condor nodes - active: diff --git a/Weekly-Reporting/export.py b/Weekly-Reporting/export.py deleted file mode 100644 index 3889fbc9..00000000 --- a/Weekly-Reporting/export.py +++ /dev/null @@ -1,232 +0,0 @@ -"""Export weekly reports statistics data to an InfluxDB bucket.""" - -from pprint import pprint -from typing import Dict, List - -from influxdb_client import Point -from pathlib import Path -import argparse -import yaml -import configparser -import logging -import datetime - -from influxdb_client.client import influxdb_client -from influxdb_client.client.write_api import SYNCHRONOUS - -time = datetime.datetime.now().isoformat() -# time = "2025-01-02T15:17:37.780483" - - -def main(args: argparse.Namespace): - """ - Main entry point to script. - :param args: Arguments provided on the command line. - """ - _check_args(args) - points = [] - if args.report_file: - points = _create_points_report(args.report_file) - elif args.inventory_file: - points = _create_points_inventory(args.inventory_file) - api_token = _get_token(args.token_file) - _write_data( - points=points, host=args.host, org=args.org, bucket=args.bucket, token=api_token - ) - - -def _check_args(args: argparse.Namespace): - """ - Check the correct arguments are provided - :param args: Argparse namespace - """ - if not args.host: - raise RuntimeError("Argument --host not given.") - if not args.org: - raise RuntimeError("Argument --org not given.") - if not args.bucket: - raise RuntimeError("Argument --bucket not given.") - if not args.token_file: - raise RuntimeError("Argument --token-file not given.") - if not args.report_file and not args.inventory_file: - raise RuntimeError("Argument --report-file or --inventory-file not given.") - if args.report_file and args.inventory_file: - raise RuntimeError( - "Argument --report-file and --inventory-file given. Only one data file can be provided." - ) - if not Path(args.token_file).is_file(): - raise RuntimeError(f"Cannot find token file at path {args.token_file}.") - if args.report_file and not Path(args.report_file).is_file(): - raise RuntimeError(f"Cannot find report file at path {args.report_file}.") - if args.inventory_file and not Path(args.inventory_file).is_file(): - raise RuntimeError(f"Cannot find inventory file at path {args.inventory_file}.") - - -def _get_token(file_path: str) -> str: - """ - Get the token from the token file. - :param file_path: File path to token file - :return: Token as string - """ - with open(Path(file_path), "r", encoding="utf-8") as file: - token = file.read().strip() - return token - - -def _create_points_report(file_path: str) -> List[Point]: - """ - Create a list of Influx points from the data. - :param file_path: Path to data file - :return: List of Points. - """ - points = [] - with open(Path(file_path), "r", encoding="utf-8") as file: - data = yaml.safe_load(file) - - for key, value in data.items(): - points += _from_key(key, value) - - return points - - -def _create_points_inventory(file_path: str) -> List[Point]: - """ - Create a list of Influx points from the data. - :param file_path: Path to data file - :return: List of Points. - """ - points = [] - config = configparser.ConfigParser() - config.read(file_path) - pprint(config) - - return points - - -def _from_key(key: str, data: Dict) -> List[Point]: - """ - Decide which create_point method to call from the key. - :param key: Section of data. For example, CPU - :param data: The values of the section - :return: List of points - """ - if key == "cpu": - return _from_cpu(data) - if key == "memory": - return _from_memory(data) - if key == "storage": - return _from_storage(data) - if key == "hv": - return _from_hv(data) - if key == "vm": - return _from_vm(data) - if key == "floating_ip": - return _from_fip(data) - if key == "virtual_worker_nodes": - return _from_vwn(data) - else: - raise RuntimeError( - f"Key {key} not supported. Please contact service maintainer." - ) - - -def _from_cpu(data: Dict) -> List[Point]: - """Extract cpu data from yaml into a Point.""" - return [ - Point("cpu") - .field("in_use", data["in_use"]) - .field("total", data["total"]) - .time(time) - ] - - -def _from_memory(data: Dict) -> List[Point]: - """Extract memory data from yaml into a Point.""" - - return [ - Point("memory") - .field("in_use", data["in_use"]) - .field("total", data["total"]) - .time(time) - ] - - -def _from_storage(data: Dict) -> List[Point]: - """Extract storage data from yaml into Points.""" - points = [] - for key, value in data.items(): - for key_2, value_2 in value.items(): - points.append(Point(key).field(key_2, value_2).time(time)) - return points - - -def _from_hv(data: Dict) -> List[Point]: - """Extract hv data from yaml into Points.""" - points = [] - points.append(Point("hv").field("active", data["active"]["active"]).time(time)) - points.append( - Point("hv").field("active_and_cpu_full", data["active"]["cpu_full"]).time(time) - ) - points.append( - Point("hv") - .field("active_and_memory_full", data["active"]["memory_full"]) - .time(time) - ) - points.append(Point("hv").field("down", data["down"]).time(time)) - points.append(Point("hv").field("disabled", data["disabled"]).time(time)) - return points - - -def _from_vm(data: Dict) -> List[Point]: - """Extract vm data from yaml into a Point.""" - return [ - ( - Point("vm") - .field("active", data["active"]) - .field("shutoff", data["shutoff"]) - .field("errored", data["errored"]) - .field("building", data["building"]) - .time(time) - ) - ] - - -def _from_fip(data: Dict) -> List[Point]: - """Extract floating ip data from yaml into a Point.""" - return [ - Point("floating_ip") - .field("in_use", data["in_use"]) - .field("total", data["total"]) - .time(time) - ] - - -def _from_vwn(data: Dict) -> List[Point]: - """Extract virtual worker nodes data from yaml into a Point.""" - return [Point("virtual_worker_nodes").field("active", data["active"]).time(time)] - - -def _write_data(points: List[Point], host: str, org: str, bucket: str, token: str): - """ - Write the data to the InfluxDB bucket. - :param points: Points to write - :param host: Host URL - :param org: InfluxDB organisation - :param bucket: InfluxDB bucket - :param token: InfluxDB API access token - """ - with influxdb_client.InfluxDBClient(url=host, token=token, org=org) as _client: - with _client.write_api(write_options=SYNCHRONOUS) as _write_api: - _write_api.write(bucket, org, points) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--host", help="InfluxDB host url with port.") - parser.add_argument("--org", help="InfluxDB organisation.") - parser.add_argument("--bucket", help="InfluxDB bucket to write to.") - parser.add_argument("--token-file", help="InfluxDB access token file path.") - parser.add_argument("--report-file", help="Report yaml file.") - parser.add_argument("--inventory-file", help="Inventory ini file.") - arguments = parser.parse_args() - main(arguments) diff --git a/Weekly-Reporting/format_raw_data.py b/Weekly-Reporting/format_raw_data.py deleted file mode 100644 index f6729cd9..00000000 --- a/Weekly-Reporting/format_raw_data.py +++ /dev/null @@ -1,79 +0,0 @@ -import yaml -from pprint import pprint - -import datetime - -with open("compute_service.yaml", "r", encoding="utf-8") as file: - compute_service = yaml.safe_load(file) - -with open("hypervisor.yaml", "r", encoding="utf-8") as file: - hypervisor = yaml.safe_load(file) - -with open("fip.yaml", "r", encoding="utf-8") as file: - fip = yaml.safe_load(file) - -with open("server.yaml", "r", encoding="utf-8") as file: - server = yaml.safe_load(file) - - -hvs_enabled = [hv for hv in compute_service if hv["Status"] == "enabled"] -hvs_up = len([hv for hv in hvs_enabled if hv["State"] == "up"]) -hvs_down = len([hv for hv in hvs_enabled if hv["State"] == "down"]) -hvs_disabled = len( - [hv for hv in hvs_enabled if hv["State"] == "up" and hv["Status"] == "disabled"] -) - -memory_used = round( - sum([hv["Memory MB Used"] for hv in hypervisor if hv["State"] == "up"]) / 1000000, 2 -) -memory_total = round( - sum([hv["Memory MB"] for hv in hypervisor if hv["State"] == "up"]) / 1000000, 2 -) - -cpu_used = sum([hv["vCPUs Used"] for hv in hypervisor if hv["State"] == "up"]) -cpu_total = sum([hv["vCPUs"] for hv in hypervisor if hv["State"] == "up"]) - -hvs_up_and_cpu_full = len( - [hv for hv in hypervisor if hv["vCPUs Used"] == hv["vCPUs"] and hv["State"] == "up"] -) -hvs_up_and_memory_full = len( - [ - hv - for hv in hypervisor - if (hv["Memory MB"] - hv["Memory MB Used"] <= 8192 and hv["State"] == "up") - ] -) - -fip_total = fip["used_ips"] -fip_used = fip["total_ips"] - -vm_active = len([vm for vm in server if vm["Status"] == "ACTIVE"]) -vm_error = len([vm for vm in server if vm["Status"] == "ERROR"]) -vm_build = len([vm for vm in server if vm["Status"] == "BUILD"]) -vm_shutoff = len([vm for vm in server if vm["Status"] == "SHUTOFF"]) - - -data = {} -data["memory"] = {"in_use": memory_used, "total": memory_total} -data["cpu"] = {"in_use": cpu_used, "total": cpu_total} -data["hv"] = { - "active": { - "active": hvs_up, - "cpu_full": hvs_up_and_cpu_full, - "memory_ful": hvs_up_and_memory_full, - }, - "down": hvs_down, - "disabled": hvs_disabled, -} -data["floating_ip"] = {"in_use": fip_used, "total": fip_total} -data["vm"] = { - "active": vm_active, - "shutoff": vm_shutoff, - "errored": vm_error, - "building": vm_build, -} - -with open( - f"data-{datetime.datetime.now().isocalendar().week}.yaml", "w", encoding="utf-8" -) as file: - yaml.dump(data, file) diff --git a/Weekly-Reporting/report.sh b/Weekly-Reporting/report.sh deleted file mode 100644 index 7c2e8533..00000000 --- a/Weekly-Reporting/report.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - - -set -e - -openstack compute service list --long -f yaml > compute_service.yaml - -openstack hypervisor list --long -f yaml > hypervisor.yaml - -openstack ip availability show 0dc30001-edfb-4137-be76-8e51f38fd650 -f yaml > fip.yaml - -openstack server list --all-projects --limit -1 -f yaml > server.yaml - - -python3 format_raw_data.py \ No newline at end of file diff --git a/Weekly-Reporting/requirements.txt b/Weekly-Reporting/requirements.txt deleted file mode 100644 index 494990ae..00000000 --- a/Weekly-Reporting/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -python-openstackclient==5.8.1 -pyyaml \ No newline at end of file