wip milestone dataframe

Author: isabelrios

__main__.py

@@ -1,9 +1,9 @@
 import argparse
 import sys
 
-#from bugz import BugzillaClient
-#from github import GithubClient
-#from jira import JiraClient
+from bugz import BugzillaClient
+from github import GithubClient
+from jira import JiraClient
 from testrail import TestRailClient
 from utils.constants import PROJECTS_MOBILE, PROJECTS_ECOSYSTEM, PLATFORM, REPORT_TYPES # noqa
 
@@ -64,7 +64,7 @@ def validate_project(platform, project, report_type):
 def main():
     args = parse_args(sys.argv[1:])
     validate_project(args.platform, args.project, args.report_type)
-    
+
     if args.report_type == 'test-case-coverage':
         h = TestRailClient()
         h.data_pump(args.project.lower())
@@ -77,13 +77,11 @@ def main():
         h.testrail_run_counts_update(args.project, num_days)
     if args.report_type == 'milestones':
         h = TestRailClient()
-        h.test_rail_milestones('59')
-    '''
+        h.test_rail_milestones()
     if args.report_type == 'issue-regression':
         h = GithubClient()
         h.github_issue_regression(args.project)
         h = GithubClient()
-
     if args.report_type == 'jira-qa-requests':
         h = JiraClient()
         h.jira_qa_requests()
@@ -93,7 +91,7 @@ def main():
     if args.report_type == 'bugzilla-qe-verify':
         h = BugzillaClient()
         h.bugzilla_qe_verify()
-    '''
+
 
 if __name__ == '__main__':
     main()

database.py

@@ -50,6 +50,11 @@ class ReportBugzillaQENeeded(Base):
 class ReportBugzillaQEVerifyCount(Base):
     __table__ = Table('report_bugzilla_qe_needed_count', Base.metadata, autoload=True) # noqa
 
+'''
+class ReportMilestones(Base):
+    __table__ = Table('report_milestones', Base.metadata, autoload=True) # noqa
+'''
+
 
 class Database:
 

testrail.py

@@ -4,15 +4,16 @@
 import pandas as pd
 
 from lib.testrail_conn import APIClient
-'''
+
 from database import (
     Database,
     Projects,
     TestSuites,
     ReportTestCaseCoverage,
+    # ReportMilestones,
     # ReportTestRunCounts
 )
-'''
+
 from utils.datetime_utils import DatetimeUtils as dt
 
 
@@ -22,8 +23,9 @@ def __init__(self):
         try:
             TESTRAIL_HOST = os.environ['TESTRAIL_HOST']
             self.client = APIClient(TESTRAIL_HOST)
-            self.client.user =  os.environ['TESTRAIL_USERNAME']
+            self.client.user = os.environ['TESTRAIL_USERNAME']
             self.client.password = os.environ['TESTRAIL_PASSWORD']
+
         except KeyError:
             print("ERROR: Missing testrail env var")
             sys.exit(1)
@@ -87,8 +89,8 @@ class TestRailClient(TestRail):
 
     def __init__(self):
         super().__init__()
-        #self.db = DatabaseTestRail()
-    '''
+        self.db = DatabaseTestRail()
+
     def data_pump(self, project='all', suite='all'):
         # call database for 'all' values
         # convert inputs to a list so we can easily
@@ -119,7 +121,7 @@ def data_pump(self, project='all', suite='all'):
                 self.testrail_coverage_update(projects_id,
                                               testrail_project_id, suite['id'])
 
-    def testrail_project_ids(self, project):
+    def testrail_project_ids(self, project='all'):
         """ Return the ids needed to be able to query the TestRail API for
         a specific test suite from a specific project
 
@@ -163,7 +165,7 @@ def testrail_coverage_update(self, projects_id,
 
         # Insert data in 'totals' array into DB
         self.db.report_test_coverage_insert(projects_id, payload)
-    
+
     def testrail_run_counts_update(self, project, num_days):
         start_date = dt.start_date(num_days)
 
@@ -187,11 +189,16 @@ def testrail_run_counts_update(self, project, num_days):
         # Insert data in 'totals' array into DB
         self.db.report_test_runs_insert(projects_id, totals)
 
-'''
-    def test_rail_milestones(self, project_id):
-        payload = self.milestones('59')
-        
-        df = pd.json_normalize(payload)
+    def test_rail_milestones(self, project='all'):
+        project_ids_list = self.testrail_project_ids(project)
+        print(project_ids_list)
+        project_id = [value[1] for value in project_ids_list]
+        milestones_all = pd.DataFrame()
+
+        for project in project_id:
+            payload = self.milestones(project)
+            df = pd.json_normalize(payload)
+            milestones_all = pd.concat([milestones_all, df], ignore_index=True)
 
         selected_columns = {
             "id": "id",
@@ -203,14 +210,39 @@ def test_rail_milestones(self, project_id):
             "completed_on": "completed_on",
             "url": "url"
         }
+
         # Select specific columns
-        df_selected = df[selected_columns.keys()]
-        df_selected['started_on'] = df_selected['started_on'].apply(dt.convert_epoch_to_datetime) # noqa
+        df_selected = milestones_all[selected_columns.keys()]
+        print("---------------")
+        print(df_selected['completed_on'].dtype)  # Float
+        print(df_selected['started_on'].dtype)  # Int
+
+        df_selected['started_on'] = pd.to_numeric(df_selected['started_on'], errors='coerce')  # noqa Ensure numeric for epoch
+        df_selected['started_on'] = pd.to_datetime(df_selected['started_on'], unit='s', errors='coerce'). # noqa
+
+        df_selected['completed_on'] = pd.to_numeric(df_selected['completed_on'], errors='coerce')  # noqa Ensure numeric for epoch
+        df_selected['completed_on'] = pd.to_datetime(df_selected['completed_on'], unit='s', errors='coerce'). # noqa
 
-        df_selected.to_csv('output.csv', index=False)
+        '''
+        These work with datatypes warnings
+        # Step 1: Replace invalid values with NaN and cast to numeric (epoch seconds)
+        df_selected.loc[:,'started_on'] = pd.to_numeric(df_selected['started_on'], errors='coerce')
+
+        # Step 2: Convert epoch timestamps to datetime64
+        df_selected.loc[:,'started_on'] = pd.to_datetime(df_selected['started_on'], unit='s', errors='coerce')
+
+        # Step 1: Replace invalid values with NaN and cast to numeric (epoch seconds)
+        df_selected.loc[:,'completed_on'] = pd.to_numeric(df_selected['completed_on'], errors='coerce')
+
+        # Step 2: Convert epoch timestamps to datetime64
+        df_selected.loc[:,'completed_on'] = pd.to_datetime(df_selected['completed_on'], unit='s', errors='coerce')
+
+        df_selected['completed_on'] = df_selected['completed_on'].dt.strftime('%Y-%m-%dT%H')
+        '''
         print(df_selected)
+        self.db.report_milestones_insert(df_selected)
+
 
-'''
 class DatabaseTestRail(Database):
 
     def __init__(self):
@@ -231,6 +263,22 @@ def test_suites_update(self, testrail_project_id,
         self.session.add(suites)
         self.session.commit()
 
+    def report_milestones_insert(sefl, payload):
+        for index, row in payload.iterrows():
+            print(row)
+            '''
+            report = ReportMilestones(milestone_id=row['id'],
+                                            project_id=row['project_id'], # noqa
+                                            milestone_name=row['name'], # noqa
+                                            milestone_start_date=row['start_on'], # noqa
+                                            milestone_complete=row['is_completed'],
+                                            milestone_end_date=row['completed_on'],
+                                            milestone_description=row['description'],
+                                            milestone_url=row['url'])
+            #self.session.add(report)
+            #self.session.commit()
+            '''
+
     def report_test_coverage_payload(self, cases):
         """given testrail data (cases), calculate test case counts by type"""
 
@@ -323,7 +371,7 @@ def report_test_runs_insert(self, project_id, payload):
             if t['testrail_completed_on']:
                 created_on = dt.convert_epoch_to_datetime(t['testrail_created_on']) # noqa
                 completed_on = dt.convert_epoch_to_datetime(t['testrail_completed_on']) # noqa
-                
+
                 report = ReportTestRunCounts(
                     projects_id=project_id,
                     testrail_run_id=t['testrail_run_id'],
@@ -333,8 +381,6 @@ def report_test_runs_insert(self, project_id, payload):
                     test_case_blocked_count=t['blocked_count'],
                     testrail_created_on=created_on,
                     testrail_completed_on=completed_on)
-                
-                
+
                 # self.session.add(report)
                 self.session.commit()
-'''

utils/datetime_utils.py

@@ -15,8 +15,11 @@ def convert_datetime_to_epoch(str_date):
         return int(t)
 
     def convert_epoch_to_datetime(int_epoch_date):
-        ts = datetime.fromtimestamp(int_epoch_date)
-        return ts.strftime(format_date)
+        if int_epoch_date == '' or int_epoch_date == 0 or int_epoch_date == 'Nan':  # noqa
+            return None
+        else:
+            ts = datetime.fromtimestamp(int_epoch_date)
+            return ts.strftime(format_date)
 
     def convert_to_utc(datetime_str):
         """Convert datetime string with timezone offset to UTC."""