Skip to content

Commit c30d37b

Browse files
committed
wip milestone dataframe
1 parent a5d16a8 commit c30d37b

File tree

4 files changed

+82
-30
lines changed

4 files changed

+82
-30
lines changed

__main__.py

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
import argparse
22
import sys
33

4-
#from bugz import BugzillaClient
5-
#from github import GithubClient
6-
#from jira import JiraClient
4+
from bugz import BugzillaClient
5+
from github import GithubClient
6+
from jira import JiraClient
77
from testrail import TestRailClient
88
from utils.constants import PROJECTS_MOBILE, PROJECTS_ECOSYSTEM, PLATFORM, REPORT_TYPES # noqa
99

@@ -64,7 +64,7 @@ def validate_project(platform, project, report_type):
6464
def main():
6565
args = parse_args(sys.argv[1:])
6666
validate_project(args.platform, args.project, args.report_type)
67-
67+
6868
if args.report_type == 'test-case-coverage':
6969
h = TestRailClient()
7070
h.data_pump(args.project.lower())
@@ -77,13 +77,11 @@ def main():
7777
h.testrail_run_counts_update(args.project, num_days)
7878
if args.report_type == 'milestones':
7979
h = TestRailClient()
80-
h.test_rail_milestones('59')
81-
'''
80+
h.test_rail_milestones()
8281
if args.report_type == 'issue-regression':
8382
h = GithubClient()
8483
h.github_issue_regression(args.project)
8584
h = GithubClient()
86-
8785
if args.report_type == 'jira-qa-requests':
8886
h = JiraClient()
8987
h.jira_qa_requests()
@@ -93,7 +91,7 @@ def main():
9391
if args.report_type == 'bugzilla-qe-verify':
9492
h = BugzillaClient()
9593
h.bugzilla_qe_verify()
96-
'''
94+
9795

9896
if __name__ == '__main__':
9997
main()

database.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,11 @@ class ReportBugzillaQENeeded(Base):
5050
class ReportBugzillaQEVerifyCount(Base):
5151
__table__ = Table('report_bugzilla_qe_needed_count', Base.metadata, autoload=True) # noqa
5252

53+
'''
54+
class ReportMilestones(Base):
55+
__table__ = Table('report_milestones', Base.metadata, autoload=True) # noqa
56+
'''
57+
5358

5459
class Database:
5560

testrail.py

Lines changed: 66 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,16 @@
44
import pandas as pd
55

66
from lib.testrail_conn import APIClient
7-
'''
7+
88
from database import (
99
Database,
1010
Projects,
1111
TestSuites,
1212
ReportTestCaseCoverage,
13+
# ReportMilestones,
1314
# ReportTestRunCounts
1415
)
15-
'''
16+
1617
from utils.datetime_utils import DatetimeUtils as dt
1718

1819

@@ -22,8 +23,9 @@ def __init__(self):
2223
try:
2324
TESTRAIL_HOST = os.environ['TESTRAIL_HOST']
2425
self.client = APIClient(TESTRAIL_HOST)
25-
self.client.user = os.environ['TESTRAIL_USERNAME']
26+
self.client.user = os.environ['TESTRAIL_USERNAME']
2627
self.client.password = os.environ['TESTRAIL_PASSWORD']
28+
2729
except KeyError:
2830
print("ERROR: Missing testrail env var")
2931
sys.exit(1)
@@ -87,8 +89,8 @@ class TestRailClient(TestRail):
8789

8890
def __init__(self):
8991
super().__init__()
90-
#self.db = DatabaseTestRail()
91-
'''
92+
self.db = DatabaseTestRail()
93+
9294
def data_pump(self, project='all', suite='all'):
9395
# call database for 'all' values
9496
# convert inputs to a list so we can easily
@@ -119,7 +121,7 @@ def data_pump(self, project='all', suite='all'):
119121
self.testrail_coverage_update(projects_id,
120122
testrail_project_id, suite['id'])
121123

122-
def testrail_project_ids(self, project):
124+
def testrail_project_ids(self, project='all'):
123125
""" Return the ids needed to be able to query the TestRail API for
124126
a specific test suite from a specific project
125127
@@ -163,7 +165,7 @@ def testrail_coverage_update(self, projects_id,
163165

164166
# Insert data in 'totals' array into DB
165167
self.db.report_test_coverage_insert(projects_id, payload)
166-
168+
167169
def testrail_run_counts_update(self, project, num_days):
168170
start_date = dt.start_date(num_days)
169171

@@ -187,11 +189,16 @@ def testrail_run_counts_update(self, project, num_days):
187189
# Insert data in 'totals' array into DB
188190
self.db.report_test_runs_insert(projects_id, totals)
189191

190-
'''
191-
def test_rail_milestones(self, project_id):
192-
payload = self.milestones('59')
193-
194-
df = pd.json_normalize(payload)
192+
def test_rail_milestones(self, project='all'):
193+
project_ids_list = self.testrail_project_ids(project)
194+
print(project_ids_list)
195+
project_id = [value[1] for value in project_ids_list]
196+
milestones_all = pd.DataFrame()
197+
198+
for project in project_id:
199+
payload = self.milestones(project)
200+
df = pd.json_normalize(payload)
201+
milestones_all = pd.concat([milestones_all, df], ignore_index=True)
195202

196203
selected_columns = {
197204
"id": "id",
@@ -203,14 +210,39 @@ def test_rail_milestones(self, project_id):
203210
"completed_on": "completed_on",
204211
"url": "url"
205212
}
213+
206214
# Select specific columns
207-
df_selected = df[selected_columns.keys()]
208-
df_selected['started_on'] = df_selected['started_on'].apply(dt.convert_epoch_to_datetime) # noqa
215+
df_selected = milestones_all[selected_columns.keys()]
216+
print("---------------")
217+
print(df_selected['completed_on'].dtype) # Float
218+
print(df_selected['started_on'].dtype) # Int
219+
220+
df_selected['started_on'] = pd.to_numeric(df_selected['started_on'], errors='coerce') # noqa Ensure numeric for epoch
221+
df_selected['started_on'] = pd.to_datetime(df_selected['started_on'], unit='s', errors='coerce'). # noqa
222+
223+
df_selected['completed_on'] = pd.to_numeric(df_selected['completed_on'], errors='coerce') # noqa Ensure numeric for epoch
224+
df_selected['completed_on'] = pd.to_datetime(df_selected['completed_on'], unit='s', errors='coerce'). # noqa
209225

210-
df_selected.to_csv('output.csv', index=False)
226+
'''
227+
These work with datatypes warnings
228+
# Step 1: Replace invalid values with NaN and cast to numeric (epoch seconds)
229+
df_selected.loc[:,'started_on'] = pd.to_numeric(df_selected['started_on'], errors='coerce')
230+
231+
# Step 2: Convert epoch timestamps to datetime64
232+
df_selected.loc[:,'started_on'] = pd.to_datetime(df_selected['started_on'], unit='s', errors='coerce')
233+
234+
# Step 1: Replace invalid values with NaN and cast to numeric (epoch seconds)
235+
df_selected.loc[:,'completed_on'] = pd.to_numeric(df_selected['completed_on'], errors='coerce')
236+
237+
# Step 2: Convert epoch timestamps to datetime64
238+
df_selected.loc[:,'completed_on'] = pd.to_datetime(df_selected['completed_on'], unit='s', errors='coerce')
239+
240+
df_selected['completed_on'] = df_selected['completed_on'].dt.strftime('%Y-%m-%dT%H')
241+
'''
211242
print(df_selected)
243+
self.db.report_milestones_insert(df_selected)
244+
212245

213-
'''
214246
class DatabaseTestRail(Database):
215247

216248
def __init__(self):
@@ -231,6 +263,22 @@ def test_suites_update(self, testrail_project_id,
231263
self.session.add(suites)
232264
self.session.commit()
233265

266+
def report_milestones_insert(sefl, payload):
267+
for index, row in payload.iterrows():
268+
print(row)
269+
'''
270+
report = ReportMilestones(milestone_id=row['id'],
271+
project_id=row['project_id'], # noqa
272+
milestone_name=row['name'], # noqa
273+
milestone_start_date=row['start_on'], # noqa
274+
milestone_complete=row['is_completed'],
275+
milestone_end_date=row['completed_on'],
276+
milestone_description=row['description'],
277+
milestone_url=row['url'])
278+
#self.session.add(report)
279+
#self.session.commit()
280+
'''
281+
234282
def report_test_coverage_payload(self, cases):
235283
"""given testrail data (cases), calculate test case counts by type"""
236284

@@ -323,7 +371,7 @@ def report_test_runs_insert(self, project_id, payload):
323371
if t['testrail_completed_on']:
324372
created_on = dt.convert_epoch_to_datetime(t['testrail_created_on']) # noqa
325373
completed_on = dt.convert_epoch_to_datetime(t['testrail_completed_on']) # noqa
326-
374+
327375
report = ReportTestRunCounts(
328376
projects_id=project_id,
329377
testrail_run_id=t['testrail_run_id'],
@@ -333,8 +381,6 @@ def report_test_runs_insert(self, project_id, payload):
333381
test_case_blocked_count=t['blocked_count'],
334382
testrail_created_on=created_on,
335383
testrail_completed_on=completed_on)
336-
337-
384+
338385
# self.session.add(report)
339386
self.session.commit()
340-
'''

utils/datetime_utils.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,11 @@ def convert_datetime_to_epoch(str_date):
1515
return int(t)
1616

1717
def convert_epoch_to_datetime(int_epoch_date):
18-
ts = datetime.fromtimestamp(int_epoch_date)
19-
return ts.strftime(format_date)
18+
if int_epoch_date == '' or int_epoch_date == 0 or int_epoch_date == 'Nan': # noqa
19+
return None
20+
else:
21+
ts = datetime.fromtimestamp(int_epoch_date)
22+
return ts.strftime(format_date)
2023

2124
def convert_to_utc(datetime_str):
2225
"""Convert datetime string with timezone offset to UTC."""

0 commit comments

Comments
 (0)