Skip to content

vs/testrail-api-ex #542

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 5 commits into from
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 26 additions & 22 deletions modules/testrail.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ def __send_request(self, method, uri, data):
try:
error = response.json()
except (
requests.exceptions.HTTPError
requests.exceptions.HTTPError
): # response.content not formatted as JSON
error = str(response.content)
raise APIError(
Expand Down Expand Up @@ -176,7 +176,7 @@ def update_test_case(self, case_id, **kwargs):
return self.client.send_post(f"update_case/{case_id}", **kwargs)

def create_test_run_on_plan_entry(
self, plan_id, entry_id, config_ids, description=None, case_ids=None
self, plan_id, entry_id, config_ids, description=None, case_ids=None
):
"""
Add a test run on an entry (subplan) associated with a plan.
Expand Down Expand Up @@ -259,12 +259,12 @@ def matching_custom_field(self, name):
return None

def create_new_plan(
self,
testrail_project_id,
name,
description=None,
milestone_id=None,
entries=None,
self,
testrail_project_id,
name,
description=None,
milestone_id=None,
entries=None,
):
"""
Create a new test plan (on a milestone).
Expand Down Expand Up @@ -297,14 +297,14 @@ def create_new_plan(
return self.client.send_post(f"/add_plan/{testrail_project_id}", payload)

def create_new_plan_entry(
self,
plan_id,
suite_id,
name=None,
description=None,
case_ids=None,
config_ids=None,
runs=None,
self,
plan_id,
suite_id,
name=None,
description=None,
case_ids=None,
config_ids=None,
runs=None,
):
"""
Create a new entry (subplan) on a plan.
Expand Down Expand Up @@ -380,12 +380,12 @@ def update_case_field(self, case_id, field_id, content):
return self.client.send_post(f"update_case/{case_id}", data)

def update_test_cases(
self,
testrail_project_id,
testrail_run_id,
testrail_suite_id,
test_case_ids=[],
status="passed",
self,
testrail_project_id,
testrail_run_id,
testrail_suite_id,
test_case_ids=[],
status="passed",
):
"""Given a project id, a run id, and a suite id, for each case given a status,
update the test objects with the correct status code"""
Expand Down Expand Up @@ -476,6 +476,10 @@ def _get_full_plan(self, plan_id):
def _get_case_fields(self):
return self.client.send_get("get_case_fields")

def get_suites(self, project_id):
"""Get all suites for project"""
return self.client.send_get(f"get_suites/{project_id}")

def _retry_api_call(self, api_call, *args, max_retries=3, delay=5):
"""
Retries the given API call up to max_retries times with a delay between attempts.
Expand Down
20 changes: 15 additions & 5 deletions modules/testrail_integration.py
Original file line number Diff line number Diff line change
Expand Up @@ -172,11 +172,21 @@ def reportable(platform_to_test=None):

def testrail_init() -> TestRail:
"""Connect to a TestRail API session"""
local = os.environ.get("TESTRAIL_BASE_URL").split("/")[2].startswith("127")
tr_session = tr.TestRail(
os.environ.get("TESTRAIL_BASE_URL"),
os.environ.get("TESTRAIL_USERNAME"),
os.environ.get("TESTRAIL_API_KEY"),
base_url = os.environ.get("TESTRAIL_BASE_URL")
username = os.environ.get("TESTRAIL_USERNAME")
api_key = os.environ.get("TESTRAIL_API_KEY")

if not base_url or not username or not api_key:
raise ValueError(
"Missing TestRail credentials. Check your api_credentials.env file."
)

local = base_url.split("/")[2].startswith("127")

tr_session = TestRail(
base_url,
username,
api_key,
local,
)
return tr_session
Expand Down
75 changes: 75 additions & 0 deletions modules/testrail_script.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
import logging
import os
from dotenv import load_dotenv
from modules.testrail_integration import testrail_init

# Set up logging
logging.basicConfig(
level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
)

# Load env file from project root
script_dir = os.path.dirname(__file__)
project_root = os.path.abspath(os.path.join(script_dir, ".."))
env_file_path = os.path.join(project_root, "api_credentials.env")
load_dotenv(dotenv_path=env_file_path)

MY_SUITES = ["Printing UI", "Profile", "Reader View"]
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the correct suite list names are: ["Printing", "Startup and Profile", "Reader View"] but I would confirm with @ben-c-at-moz

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I already updated this in my local change.

PROJECT_ID = 17

DRY_RUN = True # Set to False when performing actual run

def main():
# Read credentials from environment
base_url = os.environ.get("TESTRAIL_BASE_URL")
username = os.environ.get("TESTRAIL_USERNAME")
api_key = os.environ.get("TESTRAIL_API_KEY")

if not all([base_url, username, api_key]):
logging.error("Missing TestRail credentials. Check your .env file.")
return

logging.info(f"Loaded TestRail credentials for user: {username}")
logging.info(f"TestRail Base URL: {base_url}")

tr = testrail_init()

logging.info(f"Fetching suite IDs for suites: {MY_SUITES}")
suites = list(tr.get_suites(PROJECT_ID))
suite_ids = [
suite["id"]
for suite in suites
if suite["name"] in MY_SUITES
]
logging.info(f"Suite IDs to process: {suite_ids}")

case_ids = []
for suite_id in suite_ids:
val = tr._get_test_cases(PROJECT_ID, suite_id)
if val["size"] < val["limit"]:
matching_cases = [
case for case in val["cases"] if case["custom_automated_test_names"]
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is an interesting way to check if automation is done. Let's see if it works!

]
matching_case_ids = [case["id"] for case in matching_cases]

if DRY_RUN:
logging.info(
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Alright, run a dry run and post the ids of the cases you would have updated here. If they look good, I'll give you the go-ahead to run the script live. Thanks!

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I still have some issues with this. I will handle it ASAP.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Have you dealt with these issues?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

2025-04-08 10:20:11,390 - INFO - TestRail Base URL: https://mozilla.testrail.io
2025-04-08 10:20:11,390 - INFO - Fetching suite IDs for suites: ['Printing', 'Startup and Profile', 'Reader View']
2025-04-08 10:20:12,368 - INFO - Suite IDs to process: [73, 2119, 2126]
2025-04-08 10:20:14,092 - INFO - Suite 73 has 2 completed automated case(s).
2025-04-08 10:20:15,273 - INFO - Suite 2119 has 1 completed automated case(s).
2025-04-08 10:20:16,211 - INFO - Suite 2126 has 3 completed automated case(s).
2025-04-08 10:20:16,211 - INFO - Total completed automated cases found: 6
2025-04-08 10:20:16,211 - INFO - [DRY RUN] Would update case 965139 → custom_automation_coverage = 3
2025-04-08 10:20:16,211 - INFO - [DRY RUN] Would update case 965142 → custom_automation_coverage = 3
2025-04-08 10:20:16,211 - INFO - [DRY RUN] Would update case 130792 → custom_automation_coverage = 3
2025-04-08 10:20:16,211 - INFO - [DRY RUN] Would update case 130908 → custom_automation_coverage = 3
2025-04-08 10:20:16,211 - INFO - [DRY RUN] Would update case 130912 → custom_automation_coverage = 3
2025-04-08 10:20:16,211 - INFO - [DRY RUN] Would update case 130919 → custom_automation_coverage = 3

Process finished with exit code 0

THis is what I currently have. Is this ok?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep, that looks good, go ahead and run the script!

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I did run it. Must I do something else?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Huh, seems like that didn't change the coverage values in TR. Did you only run the dry run, or have you also run it with dry run disabled (i.e. live)? If not, go ahead and do that. If so, there may be something wrong with the script.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe the update logic wasn't added and DRY_RUN wasn't turned off

f"[DRY RUN] Suite {suite_id} would process cases: {matching_case_ids}"
)
else:
case_ids.extend(matching_case_ids)
logging.info(
f"Suite {suite_id} processed and added case IDs: {matching_case_ids}"
)
else:
logging.warning(f"Suite {suite_id} test cases exceed retrieval limit.")

if DRY_RUN:
logging.info("[DRY RUN] No actual updates performed.")
logging.info(f"[DRY RUN] Total collected case IDs (not updated): {case_ids}")
else:
logging.info(f"Total collected case IDs: {case_ids}")
# Add here actual logic for updates if necessary

if __name__ == "__main__":
main()