Modify Jira pipeline adding new issue types

.github/workflows/lint.yml

@@ -15,6 +15,6 @@ jobs:
       - name: Set up Python environment
         uses: actions/setup-python@v1
         with:
-          python-version: "3.8"
+          python-version: "3.10.16"
       - name: flake8 Lint
         uses: py-actions/flake8@v1

__main__.py

@@ -81,6 +81,7 @@ def main():
     if args.report_type == 'jira-qa-requests':
         h = JiraClient()
         h.jira_qa_requests()
+        h.jira_qa_requests_new_issue_types()
     if args.report_type == 'jira-qa-needed':
         h = JiraClient()
         h.jira_qa_needed()

database.py

@@ -39,6 +39,10 @@ class ReportJiraQARequests(Base):
     __table__ = Table('report_jira_qa_requests', Base.metadata, autoload=True) # noqa
 
 
+class ReportJIraQARequestsNewIssueType(Base):
+    __table__ = Table('report_jira_qa_requests_new_issue_types', Base.metadata, autoload=True) # noqa
+
+
 class ReportJiraQANeeded(Base):
     __table__ = Table('report_jira_qa_needed', Base.metadata, autoload=True) # noqa
 

jira.py

@@ -7,21 +7,14 @@
 from database import (
     Database,
     ReportJiraQARequests,
-    ReportJiraQANeeded
+    ReportJiraQANeeded,
+    ReportJIraQARequestsNewIssueType
 )
 from utils.datetime_utils import DatetimeUtils as dt
-from utils.constants import FILTER_ID_ALL_REQUESTS_2022, MAX_RESULT
+from utils.constants import FILTER_ID_ALL_REQUESTS_2022, FILTER_ID_NEW_ISSUE_TYPES, MAX_RESULT # noqa
+from utils.constants import JQL_QUERY, STORY_POINTS, FIREFOX_RELEASE_TRAIN, ENGINEERING_TEAM, DEFAULT_COLUMNS, COLUMNS_ISSUE_TYPE, TESTED_TRAINS # noqa
 from utils.constants import FILTER_ID_QA_NEEDED_iOS
 
-# JQL query All QA Requests since 2022 filter_id: 13856
-# Extra fields needed
-STORY_POINTS = "customfield_10037"
-FIREFOX_RELEASE_TRAIN = "customfield_10155"
-ENGINEERING_TEAM = "customfield_10134"
-DEFAULT_COLUMNS = "id,key,status,created,summary,labels,assignee,"
-
-JQL_QUERY = 'jql=filter='
-
 
 class Jira:
 
@@ -31,7 +24,6 @@ def __init__(self):
             self.client = JiraAPIClient(JIRA_HOST)
             self.client.user = os.environ['JIRA_USER']
             self.client.password = os.environ['JIRA_PASSWORD']
-
         except KeyError:
             print("ERROR: Missing jira env var")
             sys.exit(1)
@@ -45,6 +37,15 @@ def filters(self):
 
         return self.client.get_search(query)
 
+    def filters_new_issue_type(self):
+        query = JQL_QUERY + FILTER_ID_NEW_ISSUE_TYPES + '&fields=' \
+                + DEFAULT_COLUMNS + COLUMNS_ISSUE_TYPE + ',' \
+                + STORY_POINTS + ',' \
+                + FIREFOX_RELEASE_TRAIN + ',' + TESTED_TRAINS + ',' \
+                + ENGINEERING_TEAM + '&' + MAX_RESULT
+
+        return self.client.get_search(query)
+
     def filter_qa_needed(self):
         query = JQL_QUERY + FILTER_ID_QA_NEEDED_iOS + '&' + MAX_RESULT
         return self.client.get_search(query)
@@ -60,13 +61,25 @@ def jira_qa_requests(self):
         print("This is the payload returning from filter")
         print(payload)
 
-        self.db.qa_requests_delete()
+        self.db.qa_requests_delete(ReportJiraQARequests)
 
         data_frame = self.db.report_jira_qa_requests_payload(payload)
         print(data_frame)
 
         self.db.report_jira_qa_requests_insert(data_frame)
 
+    def jira_qa_requests_new_issue_types(self):
+        payload = self.filters_new_issue_type()
+        print("This is the payload returning from filter")
+        print(payload)
+
+        self.db.qa_requests_delete(ReportJIraQARequestsNewIssueType)
+
+        data_frame = self.db.report_jira_qa_requests__new_issue_types_payload(payload) # noqa
+        print(data_frame)
+
+        self.db.report_jira_qa_requests_insert_new_issue_types(data_frame)
+
     def jira_qa_needed(self):
         payload = self.filter_qa_needed()
         data_frame = self.db.report_jira_qa_needed(payload)
@@ -81,11 +94,11 @@ def __init__(self):
         super().__init__()
         self.db = Database()
 
-    def qa_requests_delete(self):
+    def qa_requests_delete(self, table):
         """ Wipe out all test suite data.
         NOTE: we'll renew this data from Testrail every session."""
         print("Delete entries from db first")
-        self.session.query(ReportJiraQARequests).delete()
+        self.session.query(table).delete()
         self.session.commit()
 
     def report_jira_qa_requests_payload(self, payload):
@@ -132,6 +145,57 @@ def report_jira_qa_requests_payload(self, payload):
 
         return df_selected
 
+    def report_jira_qa_requests__new_issue_types_payload(self, payload):
+        # Normalize the JSON data
+        self.session.query(ReportJIraQARequestsNewIssueType).delete()
+        df = pd.json_normalize(payload, sep='_')
+
+        # Check if 'jira_assignee_username' exists
+        # if not use 'alternative_assignee_emailAddress'
+        if 'fields_assignee_emailAddress' not in df.columns:
+            df['fields_assignee_emailAddress'] = df.get('fields_assignee', "None") # noqa
+        else:
+            df['fields_assignee_emailAddress'] = df['fields_assignee_emailAddress'].fillna("Not Assigned") # noqa
+
+        # Drop the alternative column if it exists
+        if 'fields_assignee' in df.columns:
+            df.drop(columns=['fields_assignee'], inplace=True)
+
+        # Select specific columns
+        selected_columns = {
+            'key': 'jira_key',
+            'fields_summary': 'jira_summary',
+            'fields_created': 'jira_created_at',
+            'fields_customfield_10155_value': 'jira_firefox_release_train',
+            'fields_customfield_10134_value': 'jira_engineering_team',
+            'fields_customfield_10037': 'jira_story_points',
+            'fields_status_name': 'jira_status',
+            'fields_assignee_emailAddress': 'jira_assignee_username',
+            'fields_labels': 'jira_labels',
+            'fields_customfield_11930': 'jira_tested_train',
+            'fields_issuetype_name': 'jira_issue_type',
+            'fields_parent_key': 'jira_parent_link'
+        }
+
+        # Select specific columns
+        df_selected = df[selected_columns.keys()]
+        print(df_selected)
+
+        # Rename columns
+        df_selected = df_selected.rename(columns=selected_columns)
+
+        df_selected['jira_created_at'] = df_selected['jira_created_at'].apply(dt.convert_to_utc) # noqa
+
+        # Join list of labels into a single string
+        df_selected['jira_labels'] = df_selected['jira_labels'].apply(lambda x: ','.join(x) if isinstance(x, list) else x) # noqa
+
+        # Convert NaN values to 0 and ensure the column is of type int
+        df_selected['jira_story_points'] = df_selected['jira_story_points'].fillna(0).astype(int) # noqa
+
+        df_selected = df_selected.where(pd.notnull(df_selected), None)
+
+        return df_selected
+
     def report_jira_qa_requests_insert(self, payload):
         print(payload)
 
@@ -149,6 +213,26 @@ def report_jira_qa_requests_insert(self, payload):
             self.session.add(report)
         self.session.commit()
 
+    def report_jira_qa_requests_insert_new_issue_types(self, payload):
+        print(payload)
+
+        for index, row in payload.iterrows():
+            print(row)
+            report = ReportJIraQARequestsNewIssueType(jira_key=row['jira_key'],
+                                                      jira_created_at=row['jira_created_at'].date(), # noqa
+                                                      jira_summary=row['jira_summary'], # noqa
+                                                      jira_firefox_release_train=row['jira_firefox_release_train'], # noqa
+                                                      jira_engineering_team=row['jira_engineering_team'], # noqa
+                                                      jira_story_points=row['jira_story_points'], # noqa
+                                                      jira_status=row['jira_status'], # noqa
+                                                      jira_assignee_username=row['jira_assignee_username'], # noqa
+                                                      jira_labels=row['jira_labels'], # noqa
+                                                      jira_tested_train=row['jira_tested_train'], # noqa
+                                                      jira_issue_type=row['jira_issue_type'], # noqa
+                                                      jira_parent_link=row['jira_parent_link']) # noqa
+            self.session.add(report)
+        self.session.commit()
+
     def report_jira_qa_needed(self, payload):
         # Normalize the JSON data
         df = pd.json_normalize(payload, sep='_')

utils/constants.py

@@ -29,9 +29,22 @@
 FILTER_ID_ALL_REQUESTS_2022 = "13856"
 MAX_RESULT = "maxResults=100"
 
+# JQL query Requests, Internal Task, Sub-Task filter_id: 14323
+FILTER_ID_NEW_ISSUE_TYPES = "14323"
+
 # JQL query All QA Needed iOS filter_id: 13789
 FILTER_ID_QA_NEEDED_iOS = "13789"
 
+# JQL Extra fields needed
+JQL_QUERY = 'jql=filter='
+
+STORY_POINTS = "customfield_10037"
+FIREFOX_RELEASE_TRAIN = "customfield_10155"
+ENGINEERING_TEAM = "customfield_10134"
+DEFAULT_COLUMNS = "id,key,status,created,summary,labels,assignee"
+COLUMNS_ISSUE_TYPE = ",issuetype,parent"
+TESTED_TRAINS = "customfield_11930"
+
 # Bugzilla queries
 BUGZILLA_URL = "bugzilla.mozilla.org"
 PRODUCTS = ["Fenix", "Focus", "GeckoView"]