From 81b8f0f767ea48467731bf337d556aa20b638201 Mon Sep 17 00:00:00 2001 From: helbashandy Date: Tue, 11 Feb 2025 09:49:25 -0800 Subject: [PATCH] Enables gsheet Integration for condo procurement - Implement fetching and caching for "Savio" and "LRC" tabs in gsheets.py - Parse decommission dates and flag alerts based on DECOMMISSION_WARNING_DAYS - Display decommission alerts in a vertical card layout - Add decommission_details view and URL route - Update README with Google Sheets setup and configuration instructions closes #546 --- README.md | 1 + bootstrap/development/docs/g-sheets-setup.md | 50 ++++++ coldfront/config/settings.py | 5 + coldfront/config/urls.py | 2 + .../templates/portal/authorized_home.html | 33 ++++ .../portal/decommission_details.html | 39 +++++ .../core/portal/templatetags/portal_tags.py | 7 + coldfront/core/portal/views.py | 59 ++++--- coldfront/core/utils/gsheets.py | 163 ++++++++++++++++++ requirements.txt | 4 + 10 files changed, 342 insertions(+), 21 deletions(-) create mode 100644 bootstrap/development/docs/g-sheets-setup.md create mode 100644 coldfront/core/portal/templates/portal/decommission_details.html create mode 100644 coldfront/core/utils/gsheets.py diff --git a/README.md b/README.md index 642ea8213..d76314efa 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ Documentation resides in a separate repository. Please request access. ### Miscellaneous Topics - [Deployment](bootstrap/ansible/README.md) +- [Google Sheets Setup](bootstrap/docs/g-sheets-setup.md) - [REST API](coldfront/api/README.md) ## License diff --git a/bootstrap/development/docs/g-sheets-setup.md b/bootstrap/development/docs/g-sheets-setup.md new file mode 100644 index 000000000..da32d3dfd --- /dev/null +++ b/bootstrap/development/docs/g-sheets-setup.md @@ -0,0 +1,50 @@ +## Google Sheets Integration + +This project uses Google Sheets to track acitivites such as: +- HPC hardware procurement and decommission information. +- TBD + + + +### Setup + +1. **Service Account & Credentials:** + - Create a Google Cloud service account with Sheets API (readonly) access. + - Set the environment variable `GOOGLE_SERVICE_ACCOUNT_JSON_PATH` to the path of your JSON credentials. + - Set `DECOMISSION_SHEET_ID` to your target spreadsheet's ID. + +2. **Django Settings:** + - **`DECOMMISSION_WARNING_DAYS`**: Days before the decommission date to trigger a warning (default: `30`). + - **`GSHEETS_CACHE_TIMEOUT`**: Cache duration in seconds (default: `86400` for 24 hours). + - **`GSHEETS_DISABLE_CACHE`**: Set to `True` during development/testing to disable caching. + +3. ** Docker-compose environment:** + - Add the following to your `docker-compose.yml` file: + ```yaml + services: + web: + environment: + - GOOGLE_SERVICE_ACCOUNT_JSON_PATH=/path/to/your/credentials.json + - DECOMISSION_SHEET_ID=your-spreadsheet-id + - GSHEETS_DISABLE_CACHE=True + # ... + ``` + +### Google Sheets Format + +- **Tabs:** The spreadsheet must include two tabs: **Savio** and **LRC**. +- **Header:** The header is on row 3 and must include at least: + - `PI Email` + - `Expected Decomission Date` (in MM/DD/YYYY format) +- **Data Rows:** Data starts from row 4. + +### HPCS Hardware Procurement Tracking + +Upon user login, the application: +- Checks both the **Savio** and **LRC** tabs for a record where `PI Email` matches the logged-in user's email. +- Parses the `Expected Decomission Date` and, if the current date is within `DECOMMISSION_WARNING_DAYS` of that date, flags the record. +- Displays decommission alerts using a card layout that lists each field vertically for clear, readable details. + +--- + +This summary should help users and developers quickly understand and set up the Google Sheets integration and the HPCS Hardware Procurement Tracking functionality. \ No newline at end of file diff --git a/coldfront/config/settings.py b/coldfront/config/settings.py index 5b871b1ea..1778a9668 100644 --- a/coldfront/config/settings.py +++ b/coldfront/config/settings.py @@ -220,6 +220,11 @@ ALLOCATION_MIN = Decimal('0.00') ALLOCATION_MAX = Decimal('100000000.00') +DECOMMISSION_WARNING_DAYS = 30 + +GSHEETS_DISABLE_CACHE = True + + # Whether to allow all jobs, bypassing all checks at job submission time. ALLOW_ALL_JOBS = False diff --git a/coldfront/config/urls.py b/coldfront/config/urls.py index 57209a9dc..6ddfd41bc 100644 --- a/coldfront/config/urls.py +++ b/coldfront/config/urls.py @@ -26,6 +26,8 @@ # path('grant/', include('coldfront.core.grant.urls')), # path('publication/', include('coldfront.core.publication.urls')), # path('research-output/', include('coldfront.core.research_output.urls')), + path('decommission-details/', portal_views.decommission_details, name='decommission_details'), + path('help', TemplateView.as_view(template_name='portal/help.html'), name='help'), ] diff --git a/coldfront/core/portal/templates/portal/authorized_home.html b/coldfront/core/portal/templates/portal/authorized_home.html index 4a0a28f19..e2a6f6643 100644 --- a/coldfront/core/portal/templates/portal/authorized_home.html +++ b/coldfront/core/portal/templates/portal/authorized_home.html @@ -1,5 +1,7 @@ {% extends "common/base.html" %} {% load common_tags %} {% block content %} +{% load portal_tags %} +

Welcome

@@ -7,6 +9,36 @@

Welcome

{% include 'portal/feedback_alert.html' %} + {% if decommission_alerts %} +
+ Decommission Notice: Your condo allocation is scheduled for decommissioning. + Please click here for details. + + + + + + + + + + + + + {% for alert in decommission_alerts %} + + + + + + + + {% endfor %} + +
Hardware TypeExpected Decomission DateStatusDepartment DivisionHardware Specification Details
{{ alert.record|get_item:"Hardware Type" }}{{ alert.record|get_item:"Expected Decomission Date" }}{{ alert.record|get_item:"Status" }}{{ alert.record|get_item:"Department Division" }}{{ alert.record|get_item:"Hardware Specification Details" }}
+
+{% endif %} +

If you would like to set up or update your access to a cluster, please complete the following steps.

First review and sign the cluster user agreement. Only then you can join a cluster project and gain access to the cluster.

@@ -113,6 +145,7 @@

Welcome

No Cluster Account {% endif %} +

diff --git a/coldfront/core/portal/templates/portal/decommission_details.html b/coldfront/core/portal/templates/portal/decommission_details.html new file mode 100644 index 000000000..79f5314ad --- /dev/null +++ b/coldfront/core/portal/templates/portal/decommission_details.html @@ -0,0 +1,39 @@ +{% extends "common/base.html" %} +{% load static %} + +{% block title %} + Condo Decommission Details +{% endblock %} + +{% block content %} +
+

Allocated Condos

+
+ {% if decommission_alerts %} + {% for alert in decommission_alerts %} +
+
+ Cluster: {{ alert.sheet }} +
+
+ {# Iterate over the record dictionary to display each field #} + {% for key, value in alert.record.items %} +
+
+ {{ key }}: +
+
+ {{ value }} +
+
+ {% endfor %} +
+
+ {% endfor %} + {% else %} +
+ No decommission alerts found! +
+ {% endif %} +
+{% endblock %} diff --git a/coldfront/core/portal/templatetags/portal_tags.py b/coldfront/core/portal/templatetags/portal_tags.py index 692cf9a2e..1f10dff85 100644 --- a/coldfront/core/portal/templatetags/portal_tags.py +++ b/coldfront/core/portal/templatetags/portal_tags.py @@ -12,3 +12,10 @@ def get_version(): @register.simple_tag def get_setting(name): return getattr(settings, name, "") + +@register.filter +def get_item(dictionary, key): + """Usage: {{ my_dict|get_item:"my_key" }}""" + if isinstance(dictionary, dict): + return dictionary.get(key, "") + return "" diff --git a/coldfront/core/portal/views.py b/coldfront/core/portal/views.py index d0bde07b3..b907f2efa 100644 --- a/coldfront/core/portal/views.py +++ b/coldfront/core/portal/views.py @@ -23,24 +23,27 @@ from coldfront.core.project.models import ProjectUserRemovalRequest from coldfront.core.project.utils import render_project_compute_usage +from coldfront.core.utils.gsheets import get_decommission_alerts_for_user +from django.contrib.auth.decorators import login_required + # from coldfront.core.publication.models import Publication # from coldfront.core.research_output.models import ResearchOutput def home(request): - context = {} if request.user.is_authenticated: template_name = 'portal/authorized_home.html' project_list = Project.objects.filter( - (Q(status__name__in=['New', 'Active', ]) & - Q(projectuser__user=request.user) & - Q(projectuser__status__name__in=['Active', 'Pending - Remove'])) + Q(status__name__in=['New', 'Active', ]) & + Q(projectuser__user=request.user) & + Q(projectuser__status__name__in=['Active', 'Pending - Remove']) ).distinct().order_by('name') - - cluster_access_attributes = AllocationUserAttribute.objects.filter(allocation_attribute_type__name='Cluster Account Status', - allocation_user__user=request.user) + cluster_access_attributes = AllocationUserAttribute.objects.filter( + allocation_attribute_type__name='Cluster Account Status', + allocation_user__user=request.user + ) access_states = {} for attribute in cluster_access_attributes: project = attribute.allocation.project @@ -49,8 +52,7 @@ def home(request): for project in project_list: project.display_status = access_states.get(project, None) - if (project.display_status is not None and - 'Active' in project.display_status): + if project.display_status is not None and 'Active' in project.display_status: context['cluster_username'] = request.user.username resource_name = get_project_compute_resource_name(project) @@ -72,21 +74,24 @@ def home(request): context['project_list'] = project_list context['allocation_list'] = allocation_list - num_join_requests = \ - ProjectUserJoinRequest.objects.filter( + num_join_requests = ProjectUserJoinRequest.objects.filter( project_user__status__name='Pending - Add', - project_user__user=request.user). \ - order_by('project_user', '-created'). \ - distinct('project_user').count() - + project_user__user=request.user + ).order_by('project_user', '-created').distinct('project_user').count() context['num_join_requests'] = num_join_requests - context['pending_removal_request_projects'] = \ - [removal_request.project_user.project.name - for removal_request in - ProjectUserRemovalRequest.objects.filter( - Q(project_user__user__username=request.user.username) & - Q(status__name='Pending'))] + context['pending_removal_request_projects'] = [ + removal_request.project_user.project.name + for removal_request in ProjectUserRemovalRequest.objects.filter( + Q(project_user__user__username=request.user.username) & + Q(status__name='Pending') + ) + ] + + # Add decommission alerts to the context. + alerts = get_decommission_alerts_for_user(request.user.email) + if alerts: + context['decommission_alerts'] = alerts else: template_name = 'portal/nonauthorized_home.html' @@ -214,3 +219,15 @@ def allocation_summary(request): context['resources_chart_data'] = resources_chart_data return render(request, 'portal/allocation_summary.html', context) + +@login_required +def decommission_details(request): + """ + Displays a page with detailed information (all fields) for each decommission record + associated with the current user's email. + """ + # Reuse the same helper function to get alerts. + alerts = get_decommission_alerts_for_user(request.user.email) + return render(request, "portal/decommission_details.html", { + "decommission_alerts": alerts + }) \ No newline at end of file diff --git a/coldfront/core/utils/gsheets.py b/coldfront/core/utils/gsheets.py new file mode 100644 index 000000000..a8b9ab3f7 --- /dev/null +++ b/coldfront/core/utils/gsheets.py @@ -0,0 +1,163 @@ +import os +from pdb import set_trace +from google.oauth2 import service_account +from googleapiclient.discovery import build +import json +import datetime +from django.conf import settings +from django.core.cache import cache + +# DECOMISSION HEADERS CONSTANTS +DECOMISSION_SHEET_HEADERS = { + "email": "PI Email", + "name": "PI Name (first last)", + "expected_decomission_date": "Expected Decomission Date", + "hardware_type": "Hardware Type", + "status": "Status", + "department_divison": "Department Division", + "hardware_specification_details": "Hardware Specification Details", +} + +def fetch_sheet_data(sheet_name, sheet_id, header_row=2): + """ + Returns the rows from your Google Sheet as a list of lists, where each sub-list is a row. + The first row is expected to be the column headers. + """ + service_account_file = os.environ.get("GOOGLE_SERVICE_ACCOUNT_JSON_PATH") + spreadsheet_id = + range_name = f"{sheet_name}!A1:Z" + + SCOPES = ["https://www.googleapis.com/auth/spreadsheets.readonly"] + creds = service_account.Credentials.from_service_account_file( + service_account_file, scopes=SCOPES + ) + service = build("sheets", "v4", credentials=creds) + + sheet = service.spreadsheets() + result = sheet.values().get( + spreadsheetId=spreadsheet_id, + range=range_name + ).execute() + + # This gives you a list of lists, each sub-list is a row + csv_data = result.get('values', []) + + # In this data the first two rows are title/empty. + # We assume that the header is in row index 2. + header = csv_data[header_row] + + return csv_data, header + +def parse_decommission_data(csv_data, headers, data_start=3): + """ + Parses the decommission data from the Google Sheet and returns a dictionary + with the PI Email as the key and the record as the value. + + """ + try: + headers.index(DECOMISSION_SHEET_HEADERS["email"]) + except ValueError: + raise ValueError(f"Header does not contain a {DECOMISSION_SHEET_HEADERS['email']} field.") + + result = {} + for row in csv_data[data_start:]: + # In case the row is shorter than the header, pad with empty strings. + if len(row) < len(headers): + for i in range(len(headers) - len(row)): + row.append("") + + # Create a dictionary mapping header names to row values. + record = {headers[i]: row[i] for i in range(len(headers))} + # Get the PI Email value and strip any extra whitespace. + + email = record.get(DECOMISSION_SHEET_HEADERS["email"], "").strip() + # If there is an email, use it as the key; + # otherwise, use the PI Name (or some other fallback). + if email: + key = email + else: + # Get a name string and replace spaces and special chars with underscores and convert to lowercase + name = record.get(DECOMISSION_SHEET_HEADERS["name"] + , "").strip().replace(" ", "_").replace("/", ".").replace("-", "_").lower() + key = f"unknown_{name}" # fallback if even PI Name is missing + + # If you expect duplicate keys (say multiple records with the same email) + # you might want to store a list of records per key. For example: + if key in result: + # If the key already exists, ensure the value is a list. + if isinstance(result[key], list): + result[key].append(record) + else: + result[key] = [result[key], record] + else: + result[key] = record + + return result, headers + +def get_cached_decomissions_sheet_data(sheet_name, cache_time=24 * 3600): + # Optionally disable caching during testing + if getattr(settings, "GSHEETS_DISABLE_CACHE", False): + csv_data,headers = fetch_sheet_data(sheet_name, os.environ.get("DECOMISSION_SHEET_ID")) + return parse_decommission_data(csv_data, headers) + + cache_key = f"gsheet_{sheet_name}" + csv_data, headers = cache.get(cache_key) + if csv_data is None: + raw_csv_data,raw_headers = fetch_sheet_data(sheet_name, os.environ.get("DECOMISSION_SHEET_ID")) + csv_data,headers = parse_decommission_data(raw_csv_data, raw_headers) + cache.set(cache_key, csv_data, cache_time) + return csv_data, headers + +def get_decommission_alerts_for_user(email, sheets=["Savio", "LRC"]): + """ + Looks in both the "Savio" and "LRC" sheets for records matching the user email. + If a record is found, it parses the 'Expected Decomission Date' (MM/DD/YYYY) and, + if today is >= (expected_date - settings.DECOMMISSION_WARNING_DAYS), + adds it as an alert. + + Returns a list of alert dictionaries. Each alert includes: + - "sheet": which sheet/tab the record came from. + - "record": a dict of the row’s data (with headers as keys). + - "expected_date": the original string for display. + """ + alerts = [] + warning_days = getattr(settings, "DECOMMISSION_WARNING_DAYS", 30) + today = datetime.date.today() + + for sheet in sheets: + decomissioned_csv_data, headers = get_cached_decomissions_sheet_data(sheet) + try: + # Ensure the required columns exist. + headers.index(DECOMISSION_SHEET_HEADERS["email"]) + except ValueError: + # Skip this sheet if the necessary columns are missing. + continue + + for email, row in decomissioned_csv_data.items(): + # Normalize row to a list of records regardless of its original type. + records = row if isinstance(row, list) else [row] + for record in records: + expected_date_str = record[DECOMISSION_SHEET_HEADERS["expected_decomission_date"] + ].strip() + if not expected_date_str: + # Skip records without a date. + continue + try: + expected_date = datetime.datetime.strptime(expected_date_str, "%m/%d/%Y").date() + except ValueError: + # Skip records with a misformatted date. + continue + + threshold_date = expected_date - datetime.timedelta(days=warning_days) + if today >= threshold_date: + alerts.append({ + "sheet": sheet, + "record": record, + "expected_date": expected_date_str, + "hardware_type": record.get(DECOMISSION_SHEET_HEADERS["hardware_type"], "").strip(), + "status": record.get(DECOMISSION_SHEET_HEADERS["status"], "").strip(), + "department_division": record.get(DECOMISSION_SHEET_HEADERS["department_divison"], "").strip(), + "hardware_specification_details": record.get(DECOMISSION_SHEET_HEADERS["hardware_specification_details"], "").strip(), + }) + + return alerts diff --git a/requirements.txt b/requirements.txt index 038956950..5ec7f7d17 100644 --- a/requirements.txt +++ b/requirements.txt @@ -57,3 +57,7 @@ tqdm==4.62.3 urllib3==1.24.2 user-agents==2.2.0 wcwidth==0.1.7 +pytest-django +google-api-python-client +google-auth-httplib2 +google-auth-oauthlib