Add Jira-Database data pump (#35)

* Add Jira-Database data pump

* adding constants

* remove argument

* adding columns to the ReportJiraQARequests

* missed a column in the report

* fix insertion and delete method

Author: isabelrios

__main__.py

@@ -2,7 +2,7 @@
 import sys
 
 from github import GithubClient
-from jira import Jira
+from jira import JiraClient
 from testrail import TestRailClient
 from utils.constants import PROJECTS_ABBREV, REPORT_TYPES
 
@@ -54,8 +54,8 @@ def main():
         h.github_issue_regression(args.project)
         h = GithubClient()
     if args.report_type == 'jira-qa-requests':
-        h = Jira()
-        h.filters()
+        h = JiraClient()
+        h.jira_qa_requests()
 
 
 if __name__ == '__main__':

database.py

@@ -35,6 +35,10 @@ class ReportGithubIssues(Base):
     __table__ = Table('report_github_issues', Base.metadata, autoload=True)
 
 
+class ReportJiraQARequests(Base):
+    __table__ = Table('report_jira_qa_requests', Base.metadata, autoload=True) # noqa
+
+
 class Database:
 
     def __init__(self):

jira.py

@@ -1,17 +1,23 @@
 import os
 import sys
 
+import pandas as pd
+
 from lib.jira_conn import JiraAPIClient
+from database import (
+    Database,
+    ReportJiraQARequests
+)
+from utils.datetime_utils import DatetimeUtils as dt
+from utils.constants import FILTER_ID_ALL_REQUESTS_2022, MAX_RESULT
 
 
 # JQL query All QA Requests since 2022 filter_id: 13856
-FILTER_ID_ALL_REQUESTS_2022 = "13856"
-# Fields needed
+# Extra fields needed
 STORY_POINTS = "customfield_10037"
 FIREFOX_RELEASE_TRAIN = "customfield_10155"
 ENGINEERING_TEAM = "customfield_10134"
 DEFAULT_COLUMNS = "id,key,status,created,summary,labels,assignee,"
-MAX_RESULT = "maxResults=100"
 
 JQL_QUERY = 'jql=filter='
 
@@ -32,6 +38,101 @@ def __init__(self):
     # API: Filters
     def filters(self):
         query = JQL_QUERY + FILTER_ID_ALL_REQUESTS_2022 + '&fields=' \
-                + DEFAULT_COLUMNS + STORY_POINTS + FIREFOX_RELEASE_TRAIN \
+                + DEFAULT_COLUMNS + ',' + STORY_POINTS + ',' \
+                + FIREFOX_RELEASE_TRAIN + ',' \
                 + ENGINEERING_TEAM + '&' + MAX_RESULT
+
         return self.client.get_search(query)
+
+
+class JiraClient(Jira):
+    def __init__(self):
+        super().__init__()
+        self.db = DatabaseJira()
+
+    def jira_qa_requests(self):
+        payload = self.filters()
+        print("This is the payload returning from filter")
+        print(payload)
+
+        self.db.qa_requests_delete()
+
+        data_frame = self.db.report_jira_qa_requests_payload(payload)
+        print(data_frame)
+
+        self.db.report_jira_qa_requests_insert(data_frame)
+
+
+class DatabaseJira(Database):
+
+    def __init__(self):
+        super().__init__()
+        self.db = Database()
+
+    def qa_requests_delete(self):
+        """ Wipe out all test suite data.
+        NOTE: we'll renew this data from Testrail every session."""
+        print("Delete entries from db first")
+        self.session.query(ReportJiraQARequests).delete()
+        self.session.commit()
+
+    def report_jira_qa_requests_payload(self, payload):
+        # Normalize the JSON data
+        df = pd.json_normalize(payload, sep='_')
+
+        # Check if 'jira_assignee_username' exists
+        # if not use 'alternative_assignee_emailAddress'
+        if 'fields_assignee_emailAddress' not in df.columns:
+            df['fields_assignee_emailAddress'] = df.get('fields_assignee', "None") # noqa
+        else:
+            df['fields_assignee_emailAddress'] = df['fields_assignee_emailAddress'].fillna("Not Assigned") # noqa
+
+        # Drop the alternative column if it exists
+        if 'fields_assignee' in df.columns:
+            df.drop(columns=['fields_assignee'], inplace=True)
+
+        # Select specific columns
+        selected_columns = {
+            'key': 'jira_key',
+            'fields_summary': 'jira_summary',
+            'fields_created': 'jira_created_at',
+            'fields_customfield_10155_value': 'jira_firefox_release_train',
+            'fields_customfield_10134_value': 'jira_engineering_team',
+            'fields_customfield_10037': 'jira_story_points',
+            'fields_status_name': 'jira_status',
+            'fields_assignee_emailAddress': 'jira_assignee_username',
+            'fields_labels': 'jira_labels'
+        }
+
+        # Select specific columns
+        df_selected = df[selected_columns.keys()]
+
+        # Rename columns
+        df_selected = df_selected.rename(columns=selected_columns)
+
+        df_selected['jira_created_at'] = df_selected['jira_created_at'].apply(dt.convert_to_utc) # noqa
+
+        # Join list of labels into a single string
+        df_selected['jira_labels'] = df_selected['jira_labels'].apply(lambda x: ','.join(x) if isinstance(x, list) else x) # noqa
+
+        # Convert NaN values to 0 and ensure the column is of type int
+        df_selected['jira_story_points'] = df_selected['jira_story_points'].fillna(0).astype(int) # noqa
+
+        return df_selected
+
+    def report_jira_qa_requests_insert(self, payload):
+        print(payload)
+
+        for index, row in payload.iterrows():
+            print(row)
+            report = ReportJiraQARequests(jira_key=row['jira_key'],
+                                          jira_created_at=row['jira_created_at'].date(), # noqa
+                                          jira_summary=row['jira_summary'], # noqa
+                                          jira_firefox_release_train=row['jira_firefox_release_train'], # noqa
+                                          jira_engineering_team=row['jira_engineering_team'], # noqa
+                                          jira_story_points=row['jira_story_points'], # noqa
+                                          jira_status=row['jira_status'], # noqa
+                                          jira_assignee_username=row['jira_assignee_username'], # noqa
+                                          jira_labels=row['jira_labels'])
+            self.session.add(report)
+        self.session.commit()

lib/jira_conn.py

@@ -37,32 +37,35 @@ def __send_request(self, method, query):
 
         # Store all results
         all_results = []
+        params = {}
 
         headers = {"Content-Type": "application/json"}
 
         # Pagination variables
-        start_at = 0
         max_results = 100
-        total = 1  # Initial value greater than start_at to enter the loop
+        total = None
+        params['startAt'] = 0
 
-        while start_at < total:
+        while True:
             # Send GET request
             response = requests.get(
                         url,
                         headers=headers,
-                        auth=HTTPBasicAuth(self.user, self.password))
-
-            if response.status_code == 200:
-                data = response.json()
-                all_results.extend(data['issues'])
-                total = data['total']  # Update total based on the response
-                start_at += max_results  # Move to the next page
-            else:
-                print(f"Failed to fetch data: {response.status_code}")
-                print(response.text)
+                        auth=HTTPBasicAuth(self.user, self.password),
+                        params=params)
+
+            data = response.json()
+            all_results.extend(data['issues'])
+            if total is None:
+                total = data['total']
+
+            # Increment the startAt parameter
+            params['startAt'] += max_results
+
+            # Check if we've retrieved all results
+            if params['startAt'] >= total:
                 break
 
         # Print the total number of issues retrieved
         print(f"Total issues retrieved: {len(all_results)}")
-        print(data)
-        return data
+        return all_results

utils/constants.py

@@ -13,3 +13,7 @@
     'issue-regression',
     'jira-qa-requests'
 ]
+
+# JQL query All QA Requests since 2022 filter_id: 13856
+FILTER_ID_ALL_REQUESTS_2022 = "13856"
+MAX_RESULT = "maxResults=100"

utils/datetime_utils.py

@@ -1,3 +1,5 @@
+import pytz
+
 from datetime import datetime, timedelta
 
 
@@ -16,6 +18,11 @@ def convert_epoch_to_datetime(int_epoch_date):
         ts = datetime.fromtimestamp(int_epoch_date)
         return ts.strftime(format_date)
 
+    def convert_to_utc(datetime_str):
+        """Convert datetime string with timezone offset to UTC."""
+        dt = datetime.strptime(datetime_str, "%Y-%m-%dT%H:%M:%S.%f%z")
+        return dt.astimezone(pytz.UTC)
+
     def start_date(num_days, end_date=''):
         """ given num_days, calculate a start_date
         given an end_date (default: now), calculate a start date num_days