diff --git a/__main__.py b/__main__.py index 04f176ec..da8b2496 100644 --- a/__main__.py +++ b/__main__.py @@ -1,9 +1,9 @@ import argparse import sys -#from bugz import BugzillaClient -#from github import GithubClient -#from jira import JiraClient +from bugz import BugzillaClient +from github import GithubClient +from jira import JiraClient from testrail import TestRailClient from utils.constants import PROJECTS_MOBILE, PROJECTS_ECOSYSTEM, PLATFORM, REPORT_TYPES # noqa @@ -64,7 +64,7 @@ def validate_project(platform, project, report_type): def main(): args = parse_args(sys.argv[1:]) validate_project(args.platform, args.project, args.report_type) - + if args.report_type == 'test-case-coverage': h = TestRailClient() h.data_pump(args.project.lower()) @@ -77,13 +77,11 @@ def main(): h.testrail_run_counts_update(args.project, num_days) if args.report_type == 'milestones': h = TestRailClient() - h.test_rail_milestones('59') - ''' + h.test_rail_milestones() if args.report_type == 'issue-regression': h = GithubClient() h.github_issue_regression(args.project) h = GithubClient() - if args.report_type == 'jira-qa-requests': h = JiraClient() h.jira_qa_requests() @@ -93,7 +91,7 @@ def main(): if args.report_type == 'bugzilla-qe-verify': h = BugzillaClient() h.bugzilla_qe_verify() - ''' + if __name__ == '__main__': main() diff --git a/database.py b/database.py index ac6cc3bf..3bb6e24d 100644 --- a/database.py +++ b/database.py @@ -50,6 +50,11 @@ class ReportBugzillaQENeeded(Base): class ReportBugzillaQEVerifyCount(Base): __table__ = Table('report_bugzilla_qe_needed_count', Base.metadata, autoload=True) # noqa +''' +class ReportMilestones(Base): + __table__ = Table('report_milestones', Base.metadata, autoload=True) # noqa +''' + class Database: diff --git a/testrail.py b/testrail.py index 7cdce3cc..26af1628 100644 --- a/testrail.py +++ b/testrail.py @@ -4,15 +4,16 @@ import pandas as pd from lib.testrail_conn import APIClient -''' + from database import ( Database, Projects, TestSuites, ReportTestCaseCoverage, + # ReportMilestones, # ReportTestRunCounts ) -''' + from utils.datetime_utils import DatetimeUtils as dt @@ -22,8 +23,9 @@ def __init__(self): try: TESTRAIL_HOST = os.environ['TESTRAIL_HOST'] self.client = APIClient(TESTRAIL_HOST) - self.client.user = os.environ['TESTRAIL_USERNAME'] + self.client.user = os.environ['TESTRAIL_USERNAME'] self.client.password = os.environ['TESTRAIL_PASSWORD'] + except KeyError: print("ERROR: Missing testrail env var") sys.exit(1) @@ -87,8 +89,8 @@ class TestRailClient(TestRail): def __init__(self): super().__init__() - #self.db = DatabaseTestRail() - ''' + self.db = DatabaseTestRail() + def data_pump(self, project='all', suite='all'): # call database for 'all' values # convert inputs to a list so we can easily @@ -119,7 +121,7 @@ def data_pump(self, project='all', suite='all'): self.testrail_coverage_update(projects_id, testrail_project_id, suite['id']) - def testrail_project_ids(self, project): + def testrail_project_ids(self, project='all'): """ Return the ids needed to be able to query the TestRail API for a specific test suite from a specific project @@ -163,7 +165,7 @@ def testrail_coverage_update(self, projects_id, # Insert data in 'totals' array into DB self.db.report_test_coverage_insert(projects_id, payload) - + def testrail_run_counts_update(self, project, num_days): start_date = dt.start_date(num_days) @@ -187,11 +189,16 @@ def testrail_run_counts_update(self, project, num_days): # Insert data in 'totals' array into DB self.db.report_test_runs_insert(projects_id, totals) -''' - def test_rail_milestones(self, project_id): - payload = self.milestones('59') - - df = pd.json_normalize(payload) + def test_rail_milestones(self, project='all'): + project_ids_list = self.testrail_project_ids(project) + print(project_ids_list) + project_id = [value[1] for value in project_ids_list] + milestones_all = pd.DataFrame() + + for project in project_id: + payload = self.milestones(project) + df = pd.json_normalize(payload) + milestones_all = pd.concat([milestones_all, df], ignore_index=True) selected_columns = { "id": "id", @@ -203,14 +210,39 @@ def test_rail_milestones(self, project_id): "completed_on": "completed_on", "url": "url" } + # Select specific columns - df_selected = df[selected_columns.keys()] - df_selected['started_on'] = df_selected['started_on'].apply(dt.convert_epoch_to_datetime) # noqa + df_selected = milestones_all[selected_columns.keys()] + print("---------------") + print(df_selected['completed_on'].dtype) # Float + print(df_selected['started_on'].dtype) # Int + + df_selected['started_on'] = pd.to_numeric(df_selected['started_on'], errors='coerce') # noqa Ensure numeric for epoch + df_selected['started_on'] = pd.to_datetime(df_selected['started_on'], unit='s', errors='coerce'). # noqa + + df_selected['completed_on'] = pd.to_numeric(df_selected['completed_on'], errors='coerce') # noqa Ensure numeric for epoch + df_selected['completed_on'] = pd.to_datetime(df_selected['completed_on'], unit='s', errors='coerce'). # noqa - df_selected.to_csv('output.csv', index=False) + ''' + These work with datatypes warnings + # Step 1: Replace invalid values with NaN and cast to numeric (epoch seconds) + df_selected.loc[:,'started_on'] = pd.to_numeric(df_selected['started_on'], errors='coerce') + + # Step 2: Convert epoch timestamps to datetime64 + df_selected.loc[:,'started_on'] = pd.to_datetime(df_selected['started_on'], unit='s', errors='coerce') + + # Step 1: Replace invalid values with NaN and cast to numeric (epoch seconds) + df_selected.loc[:,'completed_on'] = pd.to_numeric(df_selected['completed_on'], errors='coerce') + + # Step 2: Convert epoch timestamps to datetime64 + df_selected.loc[:,'completed_on'] = pd.to_datetime(df_selected['completed_on'], unit='s', errors='coerce') + + df_selected['completed_on'] = df_selected['completed_on'].dt.strftime('%Y-%m-%dT%H') + ''' print(df_selected) + self.db.report_milestones_insert(df_selected) + -''' class DatabaseTestRail(Database): def __init__(self): @@ -231,6 +263,22 @@ def test_suites_update(self, testrail_project_id, self.session.add(suites) self.session.commit() + def report_milestones_insert(sefl, payload): + for index, row in payload.iterrows(): + print(row) + ''' + report = ReportMilestones(milestone_id=row['id'], + project_id=row['project_id'], # noqa + milestone_name=row['name'], # noqa + milestone_start_date=row['start_on'], # noqa + milestone_complete=row['is_completed'], + milestone_end_date=row['completed_on'], + milestone_description=row['description'], + milestone_url=row['url']) + #self.session.add(report) + #self.session.commit() + ''' + def report_test_coverage_payload(self, cases): """given testrail data (cases), calculate test case counts by type""" @@ -323,7 +371,7 @@ def report_test_runs_insert(self, project_id, payload): if t['testrail_completed_on']: created_on = dt.convert_epoch_to_datetime(t['testrail_created_on']) # noqa completed_on = dt.convert_epoch_to_datetime(t['testrail_completed_on']) # noqa - + report = ReportTestRunCounts( projects_id=project_id, testrail_run_id=t['testrail_run_id'], @@ -333,8 +381,6 @@ def report_test_runs_insert(self, project_id, payload): test_case_blocked_count=t['blocked_count'], testrail_created_on=created_on, testrail_completed_on=completed_on) - - + # self.session.add(report) self.session.commit() -''' \ No newline at end of file diff --git a/utils/datetime_utils.py b/utils/datetime_utils.py index 995c29c8..35c5e865 100644 --- a/utils/datetime_utils.py +++ b/utils/datetime_utils.py @@ -15,8 +15,11 @@ def convert_datetime_to_epoch(str_date): return int(t) def convert_epoch_to_datetime(int_epoch_date): - ts = datetime.fromtimestamp(int_epoch_date) - return ts.strftime(format_date) + if int_epoch_date == '' or int_epoch_date == 0 or int_epoch_date == 'Nan': # noqa + return None + else: + ts = datetime.fromtimestamp(int_epoch_date) + return ts.strftime(format_date) def convert_to_utc(datetime_str): """Convert datetime string with timezone offset to UTC."""