Modify Jira pipeline adding new issue types

__main__.py

@@ -81,6 +81,7 @@ def main():
     if args.report_type == 'jira-qa-requests':
         h = JiraClient()
         h.jira_qa_requests()
+        #h.jira_qa_requests_new_issue_types()
     if args.report_type == 'jira-qa-needed':
         h = JiraClient()
         h.jira_qa_needed()

database.py

@@ -38,6 +38,9 @@ class ReportGithubIssues(Base):
 class ReportJiraQARequests(Base):
     __table__ = Table('report_jira_qa_requests', Base.metadata, autoload=True) # noqa
 
+class ReportJIraQARequestsNewIssueType(Base):
+    __table__ = Table('report_jira_qa_requests_new_issue_types', Base.metadata, autoload=True) # noqa
+
 
 class ReportJiraQANeeded(Base):
     __table__ = Table('report_jira_qa_needed', Base.metadata, autoload=True) # noqa

jira.py

@@ -7,7 +7,8 @@
 from database import (
     Database,
     ReportJiraQARequests,
-    ReportJiraQANeeded
+    ReportJiraQANeeded,
+    ReportJIraQARequestsNewIssueType
 )
 from utils.datetime_utils import DatetimeUtils as dt
 from utils.constants import FILTER_ID_ALL_REQUESTS_2022, MAX_RESULT
@@ -18,7 +19,11 @@
 STORY_POINTS = "customfield_10037"
 FIREFOX_RELEASE_TRAIN = "customfield_10155"
 ENGINEERING_TEAM = "customfield_10134"
-DEFAULT_COLUMNS = "id,key,status,created,summary,labels,assignee,"
+DEFAULT_COLUMNS = "id,key,status,created,summary,labels,assignee"
+DEFAULT_COLUMNS_ISSUE_TYPE = "id,key,status,created,summary,labels,assignee,issuetype,parent"
+TESTED_TRAINS = "customfield_11930"
+
+NEW_FILTER_ID = "14266"
 
 JQL_QUERY = 'jql=filter='
 
@@ -45,6 +50,14 @@ def filters(self):
 
         return self.client.get_search(query)
 
+    def filters_new_issue_type(self):
+        query = JQL_QUERY + NEW_FILTER_ID + '&fields=' \
+                + DEFAULT_COLUMNS_ISSUE_TYPE + ',' + STORY_POINTS + ',' \
+                + FIREFOX_RELEASE_TRAIN + ',' + TESTED_TRAINS + ',' \
+                + ENGINEERING_TEAM + '&' + MAX_RESULT
+
+        return self.client.get_search(query)
+
     def filter_qa_needed(self):
         query = JQL_QUERY + FILTER_ID_QA_NEEDED_iOS + '&' + MAX_RESULT
         return self.client.get_search(query)
@@ -67,6 +80,18 @@ def jira_qa_requests(self):
 
         self.db.report_jira_qa_requests_insert(data_frame)
 
+    def jira_qa_requests_new_issue_types(self):
+        payload = self.filters_new_issue_type()
+        print("This is the payload returning from filter")
+        print(payload)
+
+        self.db.qa_requests_delete()
+
+        data_frame = self.db.report_jira_qa_requests__new_issue_types_payload(payload)
+        print(data_frame)
+
+        self.db.report_jira_qa_requests_insert_new_issue_types(data_frame)
+
     def jira_qa_needed(self):
         payload = self.filter_qa_needed()
         data_frame = self.db.report_jira_qa_needed(payload)
@@ -132,6 +157,56 @@ def report_jira_qa_requests_payload(self, payload):
 
         return df_selected
 
+    def report_jira_qa_requests__new_issue_types_payload(self, payload):
+        # Normalize the JSON data
+        df = pd.json_normalize(payload, sep='_')
+
+        # Check if 'jira_assignee_username' exists
+        # if not use 'alternative_assignee_emailAddress'
+        if 'fields_assignee_emailAddress' not in df.columns:
+            df['fields_assignee_emailAddress'] = df.get('fields_assignee', "None") # noqa
+        else:
+            df['fields_assignee_emailAddress'] = df['fields_assignee_emailAddress'].fillna("Not Assigned") # noqa
+
+        # Drop the alternative column if it exists
+        if 'fields_assignee' in df.columns:
+            df.drop(columns=['fields_assignee'], inplace=True)
+
+        # Select specific columns
+        selected_columns = {
+            'key': 'jira_key',
+            'fields_summary': 'jira_summary',
+            'fields_created': 'jira_created_at',
+            'fields_customfield_10155_value': 'jira_firefox_release_train',
+            'fields_customfield_10134_value': 'jira_engineering_team',
+            'fields_customfield_10037': 'jira_story_points',
+            'fields_status_name': 'jira_status',
+            'fields_assignee_emailAddress': 'jira_assignee_username',
+            'fields_labels': 'jira_labels',
+            'fields_customfield_11930': 'jira_tested_train',
+            'fields_issuetype_name':'jira_issue_type',
+            'fields_parent_key': 'jira_parent_link'
+        }
+
+        # Select specific columns
+        df_selected = df[selected_columns.keys()]
+        print(df_selected)
+
+        # Rename columns
+        df_selected = df_selected.rename(columns=selected_columns)
+
+        df_selected['jira_created_at'] = df_selected['jira_created_at'].apply(dt.convert_to_utc) # noqa
+
+        # Join list of labels into a single string
+        df_selected['jira_labels'] = df_selected['jira_labels'].apply(lambda x: ','.join(x) if isinstance(x, list) else x) # noqa
+
+        # Convert NaN values to 0 and ensure the column is of type int
+        df_selected['jira_story_points'] = df_selected['jira_story_points'].fillna(0).astype(int) # noqa
+
+        df_selected = df_selected.where(pd.notnull(df_selected), None)
+
+        return df_selected
+
     def report_jira_qa_requests_insert(self, payload):
         print(payload)
 
@@ -149,6 +224,27 @@ def report_jira_qa_requests_insert(self, payload):
             self.session.add(report)
         self.session.commit()
 
+    def report_jira_qa_requests_insert_new_issue_types(self, payload):
+        print(payload)
+
+        for index, row in payload.iterrows():
+            print(row)
+            report = ReportJIraQARequestsNewIssueType(jira_key=row['jira_key'],
+                                          jira_created_at=row['jira_created_at'].date(), # noqa
+                                          jira_summary=row['jira_summary'], # noqa
+                                          jira_firefox_release_train=row['jira_firefox_release_train'], # noqa
+                                          jira_engineering_team=row['jira_engineering_team'], # noqa
+                                          jira_story_points=row['jira_story_points'], # noqa
+                                          jira_status=row['jira_status'], # noqa
+                                          jira_assignee_username=row['jira_assignee_username'], # noqa
+                                          jira_labels=row['jira_labels'],
+                                          jira_tested_train=row['jira_tested_train'],
+                                          jira_issue_type=row['jira_issue_type'],
+                                          jira_parent_link=row['jira_parent_link']
+                                    )
+            self.session.add(report)
+        self.session.commit()
+
     def report_jira_qa_needed(self, payload):
         # Normalize the JSON data
         df = pd.json_normalize(payload, sep='_')