From fb4ce98a94b1fb64d1c48828ec9b397344a712f2 Mon Sep 17 00:00:00 2001 From: isabelrios Date: Tue, 9 Jul 2024 11:56:22 +0200 Subject: [PATCH] Add Jira-Database data pump --- __main__.py | 6 +-- database.py | 4 ++ jira.py | 88 +++++++++++++++++++++++++++++++++++++++-- lib/jira_conn.py | 34 +++++++++------- utils/datetime_utils.py | 7 ++++ 5 files changed, 118 insertions(+), 21 deletions(-) diff --git a/__main__.py b/__main__.py index 9f247f72..1e42f369 100644 --- a/__main__.py +++ b/__main__.py @@ -2,7 +2,7 @@ import sys from github import GithubClient -from jira import Jira +from jira import JiraClient from testrail import TestRailClient from utils.constants import PROJECTS_ABBREV, REPORT_TYPES @@ -54,8 +54,8 @@ def main(): h.github_issue_regression(args.project) h = GithubClient() if args.report_type == 'jira-qa-requests': - h = Jira() - h.filters() + h = JiraClient() + h.jira_qa_requests() if __name__ == '__main__': diff --git a/database.py b/database.py index ba8a1359..51fa56c0 100644 --- a/database.py +++ b/database.py @@ -35,6 +35,10 @@ class ReportGithubIssues(Base): __table__ = Table('report_github_issues', Base.metadata, autoload=True) +class ReportJiraQARequests(Base): + __table__ = Table('report_jira_qa_requests', Base.metadata, autoload=True) # noqa + + class Database: def __init__(self): diff --git a/jira.py b/jira.py index 58562539..53c9c39b 100644 --- a/jira.py +++ b/jira.py @@ -1,8 +1,15 @@ import os import sys +import json -from lib.jira_conn import JiraAPIClient +import pandas as pd +from lib.jira_conn import JiraAPIClient +from database import ( + Database, + ReportJiraQARequests +) +from utils.datetime_utils import DatetimeUtils as dt # JQL query All QA Requests since 2022 filter_id: 13856 FILTER_ID_ALL_REQUESTS_2022 = "13856" @@ -32,6 +39,81 @@ def __init__(self): # API: Filters def filters(self): query = JQL_QUERY + FILTER_ID_ALL_REQUESTS_2022 + '&fields=' \ - + DEFAULT_COLUMNS + STORY_POINTS + FIREFOX_RELEASE_TRAIN \ - + ENGINEERING_TEAM + '&' + MAX_RESULT + + DEFAULT_COLUMNS + ',' + STORY_POINTS + ',' + FIREFOX_RELEASE_TRAIN \ + + ',' + ENGINEERING_TEAM + '&' + MAX_RESULT + return self.client.get_search(query) + + +class JiraClient(Jira): + def __init__(self): + super().__init__() + self.db = DatabaseJira() + + def jira_qa_requests(self): + payload = self.filters() + + data_frame = self.db.report_jira_qa_requests_payload(payload) + print(data_frame) + + self.db.report_jira_qa_requests_insert(data_frame) + + +class DatabaseJira(Database): + + def __init__(self): + super().__init__() + self.db = Database() + + def report_jira_qa_requests_payload(self, payload): + # Normalize the JSON data + df = pd.json_normalize(payload, sep='_') + + # Check if 'jira_assignee_username' exists, if not use 'alternative_assignee_emailAddress' + if 'fields_assignee_emailAddress' not in df.columns: + df['fields_assignee_emailAddress'] = df.get('fields_assignee', "None") + else: + df['fields_assignee_emailAddress'] = df['fields_assignee_emailAddress'].fillna("Not Assigned") + + # Drop the alternative column if it exists + if 'fields_assignee' in df.columns: + df.drop(columns=['fields_assignee'], inplace=True) + + # Select specific columns + selected_columns = { + 'key': 'jira_key', + 'fields_summary': 'jira_summary', + 'fields_created': 'jira_created_at', + 'fields_customfield_10155_value': 'jira_firefox_release_train', + 'fields_customfield_10134_value': 'jira_engineering_team', + 'fields_customfield_10037': 'jira_story_points', + 'fields_status_name': 'jira_status', + 'fields_assignee_emailAddress': 'jira_assignee_username', + 'fields_labels': 'jira_labels' + } + + #Select specific columns + df_selected = df[selected_columns.keys()] + + # Rename columns + df_selected = df_selected.rename(columns=selected_columns) + + df_selected['jira_created_at'] = df_selected['jira_created_at'].apply(dt.convert_to_utc) + + # Join list of labels into a single string + df_selected['jira_labels'] = df_selected['jira_labels'].apply(lambda x: ','.join(x) if isinstance(x, list) else x) + + return df_selected + + + def report_jira_qa_requests_insert(self, payload): + pass + # TBD + # This is the way used in testrail.py + # report = ReportJiraQARequests(payload) + + # This is the only way working locally to insert data + # payload.to_sql('report_jira_qa_requests', con=engine, if_exists='append', index=False) + + # session.add(report) + # session.commit() diff --git a/lib/jira_conn.py b/lib/jira_conn.py index 2ba79082..7e8740b0 100644 --- a/lib/jira_conn.py +++ b/lib/jira_conn.py @@ -7,6 +7,7 @@ Copyright Atlassian developer. See license.md for details. """ +import json import requests from requests.auth import HTTPBasicAuth @@ -37,32 +38,35 @@ def __send_request(self, method, query): # Store all results all_results = [] + params = {} headers = {"Content-Type": "application/json"} # Pagination variables - start_at = 0 max_results = 100 - total = 1 # Initial value greater than start_at to enter the loop + total = None + params['startAt'] = 0 - while start_at < total: + while True: # Send GET request response = requests.get( url, headers=headers, - auth=HTTPBasicAuth(self.user, self.password)) - - if response.status_code == 200: - data = response.json() - all_results.extend(data['issues']) - total = data['total'] # Update total based on the response - start_at += max_results # Move to the next page - else: - print(f"Failed to fetch data: {response.status_code}") - print(response.text) + auth=HTTPBasicAuth(self.user, self.password), + params=params) + + data = response.json() + all_results.extend(data['issues']) + if total is None: + total = data['total'] + + # Increment the startAt parameter + params['startAt'] += max_results + + # Check if we've retrieved all results + if params['startAt'] >= total: break # Print the total number of issues retrieved print(f"Total issues retrieved: {len(all_results)}") - print(data) - return data + return all_results diff --git a/utils/datetime_utils.py b/utils/datetime_utils.py index ebc224e9..995c29c8 100644 --- a/utils/datetime_utils.py +++ b/utils/datetime_utils.py @@ -1,3 +1,5 @@ +import pytz + from datetime import datetime, timedelta @@ -16,6 +18,11 @@ def convert_epoch_to_datetime(int_epoch_date): ts = datetime.fromtimestamp(int_epoch_date) return ts.strftime(format_date) + def convert_to_utc(datetime_str): + """Convert datetime string with timezone offset to UTC.""" + dt = datetime.strptime(datetime_str, "%Y-%m-%dT%H:%M:%S.%f%z") + return dt.astimezone(pytz.UTC) + def start_date(num_days, end_date=''): """ given num_days, calculate a start_date given an end_date (default: now), calculate a start date num_days