Add Jira-Database data pump

__main__.py

@@ -2,7 +2,7 @@
 import sys
 
 from github import GithubClient
-from jira import Jira
+from jira import JiraClient
 from testrail import TestRailClient
 from utils.constants import PROJECTS_ABBREV, REPORT_TYPES
 
@@ -54,8 +54,8 @@ def main():
         h.github_issue_regression(args.project)
         h = GithubClient()
     if args.report_type == 'jira-qa-requests':
-        h = Jira()
-        h.filters()
+        h = JiraClient()
+        h.jira_qa_requests()
 
 
 if __name__ == '__main__':

database.py

@@ -35,6 +35,10 @@ class ReportGithubIssues(Base):
     __table__ = Table('report_github_issues', Base.metadata, autoload=True)
 
 
+class ReportJiraQARequests(Base):
+    __table__ = Table('report_jira_qa_requests', Base.metadata, autoload=True) # noqa
+
+
 class Database:
 
     def __init__(self):

jira.py

@@ -1,8 +1,15 @@
 import os
 import sys
+import json
 
-from lib.jira_conn import JiraAPIClient
+import pandas as pd
 
+from lib.jira_conn import JiraAPIClient
+from database import (
+    Database,
+    ReportJiraQARequests
+)
+from utils.datetime_utils import DatetimeUtils as dt
 
 # JQL query All QA Requests since 2022 filter_id: 13856
 FILTER_ID_ALL_REQUESTS_2022 = "13856"
@@ -32,6 +39,81 @@ def __init__(self):
     # API: Filters
     def filters(self):
         query = JQL_QUERY + FILTER_ID_ALL_REQUESTS_2022 + '&fields=' \
-                + DEFAULT_COLUMNS + STORY_POINTS + FIREFOX_RELEASE_TRAIN \
-                + ENGINEERING_TEAM + '&' + MAX_RESULT
+                + DEFAULT_COLUMNS + ',' + STORY_POINTS + ',' + FIREFOX_RELEASE_TRAIN \
+                + ',' + ENGINEERING_TEAM + '&' + MAX_RESULT
+
         return self.client.get_search(query)
+
+
+class JiraClient(Jira):
+    def __init__(self):
+        super().__init__()
+        self.db = DatabaseJira()
+
+    def jira_qa_requests(self):
+        payload = self.filters()
+
+        data_frame = self.db.report_jira_qa_requests_payload(payload)
+        print(data_frame)
+
+        self.db.report_jira_qa_requests_insert(data_frame)
+
+
+class DatabaseJira(Database):
+
+    def __init__(self):
+        super().__init__()
+        self.db = Database()
+
+    def report_jira_qa_requests_payload(self, payload):
+        # Normalize the JSON data
+        df = pd.json_normalize(payload, sep='_')
+
+        # Check if 'jira_assignee_username' exists, if not use 'alternative_assignee_emailAddress'
+        if 'fields_assignee_emailAddress' not in df.columns:
+            df['fields_assignee_emailAddress'] = df.get('fields_assignee', "None")
+        else:
+            df['fields_assignee_emailAddress'] = df['fields_assignee_emailAddress'].fillna("Not Assigned")
+
+        # Drop the alternative column if it exists
+        if 'fields_assignee' in df.columns:
+            df.drop(columns=['fields_assignee'], inplace=True)
+
+        # Select specific columns
+        selected_columns = {
+            'key': 'jira_key',
+            'fields_summary': 'jira_summary',
+            'fields_created': 'jira_created_at',
+            'fields_customfield_10155_value': 'jira_firefox_release_train',
+            'fields_customfield_10134_value': 'jira_engineering_team',
+            'fields_customfield_10037': 'jira_story_points',
+            'fields_status_name': 'jira_status',
+            'fields_assignee_emailAddress': 'jira_assignee_username',
+            'fields_labels': 'jira_labels'
+        }
+
+        #Select specific columns
+        df_selected = df[selected_columns.keys()]
+
+        # Rename columns
+        df_selected = df_selected.rename(columns=selected_columns)
+
+        df_selected['jira_created_at'] = df_selected['jira_created_at'].apply(dt.convert_to_utc)
+
+        # Join list of labels into a single string
+        df_selected['jira_labels'] = df_selected['jira_labels'].apply(lambda x: ','.join(x) if isinstance(x, list) else x)
+
+        return df_selected
+
+
+    def report_jira_qa_requests_insert(self, payload):
+        pass
+        # TBD
+        # This is the way used in testrail.py
+        # report = ReportJiraQARequests(payload)
+
+        # This is the only way working locally to insert data
+        # payload.to_sql('report_jira_qa_requests', con=engine, if_exists='append', index=False)
+
+        # session.add(report)
+        # session.commit()

lib/jira_conn.py

@@ -7,6 +7,7 @@
 
 Copyright Atlassian developer. See license.md for details.
 """
+import json
 import requests
 
 from requests.auth import HTTPBasicAuth
@@ -37,32 +38,35 @@ def __send_request(self, method, query):
 
         # Store all results
         all_results = []
+        params = {}
 
         headers = {"Content-Type": "application/json"}
 
         # Pagination variables
-        start_at = 0
         max_results = 100
-        total = 1  # Initial value greater than start_at to enter the loop
+        total = None
+        params['startAt'] = 0
 
-        while start_at < total:
+        while True:
             # Send GET request
             response = requests.get(
                         url,
                         headers=headers,
-                        auth=HTTPBasicAuth(self.user, self.password))
-
-            if response.status_code == 200:
-                data = response.json()
-                all_results.extend(data['issues'])
-                total = data['total']  # Update total based on the response
-                start_at += max_results  # Move to the next page
-            else:
-                print(f"Failed to fetch data: {response.status_code}")
-                print(response.text)
+                        auth=HTTPBasicAuth(self.user, self.password),
+                        params=params)
+
+            data = response.json()
+            all_results.extend(data['issues'])
+            if total is None:
+                total = data['total']
+
+            # Increment the startAt parameter
+            params['startAt'] += max_results
+
+            # Check if we've retrieved all results
+            if params['startAt'] >= total:
                 break
 
         # Print the total number of issues retrieved
         print(f"Total issues retrieved: {len(all_results)}")
-        print(data)
-        return data
+        return all_results

utils/datetime_utils.py

@@ -1,3 +1,5 @@
+import pytz
+
 from datetime import datetime, timedelta
 
 
@@ -16,6 +18,11 @@ def convert_epoch_to_datetime(int_epoch_date):
         ts = datetime.fromtimestamp(int_epoch_date)
         return ts.strftime(format_date)
 
+    def convert_to_utc(datetime_str):
+        """Convert datetime string with timezone offset to UTC."""
+        dt = datetime.strptime(datetime_str, "%Y-%m-%dT%H:%M:%S.%f%z")
+        return dt.astimezone(pytz.UTC)
+
     def start_date(num_days, end_date=''):
         """ given num_days, calculate a start_date
         given an end_date (default: now), calculate a start date num_days