Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor(anta.tests): Nicer result failure messages Service and SNMP test module  #1041

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion anta/input_models/services.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ def __str__(self) -> str:
--------
Server 10.0.0.1 (VRF: default, Priority: 1)
"""
return f"Server {self.server_address} (VRF: {self.vrf}, Priority: {self.priority})"
return f"Server {self.server_address} VRF: {self.vrf} Priority: {self.priority}"


class ErrdisableRecovery(BaseModel):
Expand Down
2 changes: 1 addition & 1 deletion anta/tests/services.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ def test(self) -> None:
hostname = self.instance_commands[0].json_output["hostname"]

if hostname != self.inputs.hostname:
self.result.is_failure(f"Expected `{self.inputs.hostname}` as the hostname, but found `{hostname}` instead.")
self.result.is_failure(f"Incorrect Hostname - Expected: {self.inputs.hostname} Actual: {hostname}")
else:
self.result.is_success()

Expand Down
58 changes: 24 additions & 34 deletions anta/tests/snmp.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@


class VerifySnmpStatus(AntaTest):
"""Verifies whether the SNMP agent is enabled in a specified VRF.
"""Verifies if the SNMP agent is enabled.

Expected Results
----------------
Expand All @@ -37,7 +37,6 @@ class VerifySnmpStatus(AntaTest):
```
"""

description = "Verifies if the SNMP agent is enabled."
categories: ClassVar[list[str]] = ["snmp"]
commands: ClassVar[list[AntaCommand | AntaTemplate]] = [AntaCommand(command="show snmp", revision=1)]

Expand All @@ -50,15 +49,14 @@ class Input(AntaTest.Input):
@AntaTest.anta_test
def test(self) -> None:
"""Main test function for VerifySnmpStatus."""
self.result.is_success()
command_output = self.instance_commands[0].json_output
if command_output["enabled"] and self.inputs.vrf in command_output["vrfs"]["snmpVrfs"]:
self.result.is_success()
else:
self.result.is_failure(f"SNMP agent disabled in vrf {self.inputs.vrf}")
if not (command_output["enabled"] and self.inputs.vrf in command_output["vrfs"]["snmpVrfs"]):
self.result.is_failure(f"VRF: {self.inputs.vrf} - SNMP agent disabled")


class VerifySnmpIPv4Acl(AntaTest):
"""Verifies if the SNMP agent has the right number IPv4 ACL(s) configured for a specified VRF.
"""Verifies if the SNMP agent has IPv4 ACL(s) configured.

Expected Results
----------------
Expand All @@ -75,7 +73,6 @@ class VerifySnmpIPv4Acl(AntaTest):
```
"""

description = "Verifies if the SNMP agent has IPv4 ACL(s) configured."
categories: ClassVar[list[str]] = ["snmp"]
commands: ClassVar[list[AntaCommand | AntaTemplate]] = [AntaCommand(command="show snmp ipv4 access-list summary", revision=1)]

Expand All @@ -90,23 +87,22 @@ class Input(AntaTest.Input):
@AntaTest.anta_test
def test(self) -> None:
"""Main test function for VerifySnmpIPv4Acl."""
self.result.is_success()
command_output = self.instance_commands[0].json_output
ipv4_acl_list = command_output["ipAclList"]["aclList"]
ipv4_acl_number = len(ipv4_acl_list)
if ipv4_acl_number != self.inputs.number:
self.result.is_failure(f"Expected {self.inputs.number} SNMP IPv4 ACL(s) in vrf {self.inputs.vrf} but got {ipv4_acl_number}")
self.result.is_failure(f"VRF: {self.inputs.vrf} - Incorrect SNMP IPv4 ACL(s) - Expected: {self.inputs.number} Actual: {ipv4_acl_number}")
return

not_configured_acl = [acl["name"] for acl in ipv4_acl_list if self.inputs.vrf not in acl["configuredVrfs"] or self.inputs.vrf not in acl["activeVrfs"]]

if not_configured_acl:
self.result.is_failure(f"SNMP IPv4 ACL(s) not configured or active in vrf {self.inputs.vrf}: {not_configured_acl}")
else:
self.result.is_success()
self.result.is_failure(f"VRF: {self.inputs.vrf} - Following SNMP IPv4 ACL(s) not configured or active: {', '.join(not_configured_acl)}")


class VerifySnmpIPv6Acl(AntaTest):
"""Verifies if the SNMP agent has the right number IPv6 ACL(s) configured for a specified VRF.
"""Verifies if the SNMP agent has IPv6 ACL(s) configured.

Expected Results
----------------
Expand All @@ -123,7 +119,6 @@ class VerifySnmpIPv6Acl(AntaTest):
```
"""

description = "Verifies if the SNMP agent has IPv6 ACL(s) configured."
categories: ClassVar[list[str]] = ["snmp"]
commands: ClassVar[list[AntaCommand | AntaTemplate]] = [AntaCommand(command="show snmp ipv6 access-list summary", revision=1)]

Expand All @@ -139,18 +134,17 @@ class Input(AntaTest.Input):
def test(self) -> None:
"""Main test function for VerifySnmpIPv6Acl."""
command_output = self.instance_commands[0].json_output
self.result.is_success()
ipv6_acl_list = command_output["ipv6AclList"]["aclList"]
ipv6_acl_number = len(ipv6_acl_list)
if ipv6_acl_number != self.inputs.number:
self.result.is_failure(f"Expected {self.inputs.number} SNMP IPv6 ACL(s) in vrf {self.inputs.vrf} but got {ipv6_acl_number}")
self.result.is_failure(f"VRF: {self.inputs.vrf} - Incorrect SNMP IPv6 ACL(s) - Expected: {self.inputs.number} Actual: {ipv6_acl_number}")
return

acl_not_configured = [acl["name"] for acl in ipv6_acl_list if self.inputs.vrf not in acl["configuredVrfs"] or self.inputs.vrf not in acl["activeVrfs"]]

if acl_not_configured:
self.result.is_failure(f"SNMP IPv6 ACL(s) not configured or active in vrf {self.inputs.vrf}: {acl_not_configured}")
else:
self.result.is_success()
self.result.is_failure(f"VRF: {self.inputs.vrf} - Following SNMP IPv6 ACL(s) not configured or active: {', '.join(acl_not_configured)}")


class VerifySnmpLocation(AntaTest):
Expand Down Expand Up @@ -182,16 +176,15 @@ class Input(AntaTest.Input):
@AntaTest.anta_test
def test(self) -> None:
"""Main test function for VerifySnmpLocation."""
self.result.is_success()
# Verifies the SNMP location is configured.
if not (location := get_value(self.instance_commands[0].json_output, "location.location")):
self.result.is_failure("SNMP location is not configured.")
return

# Verifies the expected SNMP location.
if location != self.inputs.location:
self.result.is_failure(f"Expected `{self.inputs.location}` as the location, but found `{location}` instead.")
else:
self.result.is_success()
self.result.is_failure(f"Incorrect SNMP location - Expected: {self.inputs.location} Actual: {location}")


class VerifySnmpContact(AntaTest):
Expand Down Expand Up @@ -223,16 +216,15 @@ class Input(AntaTest.Input):
@AntaTest.anta_test
def test(self) -> None:
"""Main test function for VerifySnmpContact."""
self.result.is_success()
# Verifies the SNMP contact is configured.
if not (contact := get_value(self.instance_commands[0].json_output, "contact.contact")):
self.result.is_failure("SNMP contact is not configured.")
return

# Verifies the expected SNMP contact.
if contact != self.inputs.contact:
self.result.is_failure(f"Expected `{self.inputs.contact}` as the contact, but found `{contact}` instead.")
else:
self.result.is_success()
self.result.is_failure(f"Incorrect SNMP contact - Expected: {self.inputs.contact} Actual: {contact}")


class VerifySnmpPDUCounters(AntaTest):
Expand Down Expand Up @@ -269,6 +261,7 @@ class Input(AntaTest.Input):
@AntaTest.anta_test
def test(self) -> None:
"""Main test function for VerifySnmpPDUCounters."""
self.result.is_success()
snmp_pdus = self.inputs.pdus
command_output = self.instance_commands[0].json_output

Expand All @@ -281,13 +274,11 @@ def test(self) -> None:
if not snmp_pdus:
snmp_pdus = list(get_args(SnmpPdu))

failures = {pdu: value for pdu in snmp_pdus if (value := pdu_counters.get(pdu, "Not Found")) == "Not Found" or value == 0}
failures = {pdu for pdu in snmp_pdus if (value := pdu_counters.get(pdu, "Not Found")) == "Not Found" or value == 0}

# Check if any failures
if not failures:
self.result.is_success()
else:
self.result.is_failure(f"The following SNMP PDU counters are not found or have zero PDU counters:\n{failures}")
if failures:
self.result.is_failure(f"The following SNMP PDU counters are not found or have zero PDU counters: {', '.join(sorted(failures))}")


class VerifySnmpErrorCounters(AntaTest):
Expand Down Expand Up @@ -323,6 +314,7 @@ class Input(AntaTest.Input):
@AntaTest.anta_test
def test(self) -> None:
"""Main test function for VerifySnmpErrorCounters."""
self.result.is_success()
error_counters = self.inputs.error_counters
command_output = self.instance_commands[0].json_output

Expand All @@ -335,13 +327,11 @@ def test(self) -> None:
if not error_counters:
error_counters = list(get_args(SnmpErrorCounter))

error_counters_not_ok = {counter: value for counter in error_counters if (value := snmp_counters.get(counter))}
error_counters_not_ok = {counter for counter in error_counters if snmp_counters.get(counter)}

# Check if any failures
if not error_counters_not_ok:
self.result.is_success()
else:
self.result.is_failure(f"The following SNMP error counters are not found or have non-zero error counters:\n{error_counters_not_ok}")
if error_counters_not_ok:
self.result.is_failure(f"The following SNMP error counters are not found or have non-zero error counters: {', '.join(sorted(error_counters_not_ok))}")


class VerifySnmpHostLogging(AntaTest):
Expand Down
10 changes: 5 additions & 5 deletions tests/units/anta_tests/test_services.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
"inputs": {"hostname": "s1-spine1"},
"expected": {
"result": "failure",
"messages": ["Expected `s1-spine1` as the hostname, but found `s1-spine2` instead."],
"messages": ["Incorrect Hostname - Expected: s1-spine1 Actual: s1-spine2"],
},
},
{
Expand Down Expand Up @@ -88,7 +88,7 @@
},
"expected": {
"result": "failure",
"messages": ["Server 10.14.0.10 (VRF: default, Priority: 0) - Not configured", "Server 10.14.0.21 (VRF: MGMT, Priority: 1) - Not configured"],
"messages": ["Server 10.14.0.10 VRF: default Priority: 0 - Not configured", "Server 10.14.0.21 VRF: MGMT Priority: 1 - Not configured"],
},
},
{
Expand All @@ -109,9 +109,9 @@
"expected": {
"result": "failure",
"messages": [
"Server 10.14.0.1 (VRF: CS, Priority: 0) - Incorrect priority - Priority: 1",
"Server 10.14.0.11 (VRF: default, Priority: 0) - Not configured",
"Server 10.14.0.110 (VRF: MGMT, Priority: 0) - Not configured",
"Server 10.14.0.1 VRF: CS Priority: 0 - Incorrect priority - Priority: 1",
"Server 10.14.0.11 VRF: default Priority: 0 - Not configured",
"Server 10.14.0.110 VRF: MGMT Priority: 0 - Not configured",
],
},
},
Expand Down
24 changes: 11 additions & 13 deletions tests/units/anta_tests/test_snmp.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,14 @@
"test": VerifySnmpStatus,
"eos_data": [{"vrfs": {"snmpVrfs": ["default"]}, "enabled": True}],
"inputs": {"vrf": "MGMT"},
"expected": {"result": "failure", "messages": ["SNMP agent disabled in vrf MGMT"]},
"expected": {"result": "failure", "messages": ["VRF: MGMT - SNMP agent disabled"]},
},
{
"name": "failure-disabled",
"test": VerifySnmpStatus,
"eos_data": [{"vrfs": {"snmpVrfs": ["default"]}, "enabled": False}],
"inputs": {"vrf": "default"},
"expected": {"result": "failure", "messages": ["SNMP agent disabled in vrf default"]},
"expected": {"result": "failure", "messages": ["VRF: default - SNMP agent disabled"]},
},
{
"name": "success",
Expand All @@ -57,14 +57,14 @@
"test": VerifySnmpIPv4Acl,
"eos_data": [{"ipAclList": {"aclList": []}}],
"inputs": {"number": 1, "vrf": "MGMT"},
"expected": {"result": "failure", "messages": ["Expected 1 SNMP IPv4 ACL(s) in vrf MGMT but got 0"]},
"expected": {"result": "failure", "messages": ["VRF: MGMT - Incorrect SNMP IPv4 ACL(s) - Expected: 1 Actual: 0"]},
},
{
"name": "failure-wrong-vrf",
"test": VerifySnmpIPv4Acl,
"eos_data": [{"ipAclList": {"aclList": [{"type": "Ip4Acl", "name": "ACL_IPV4_SNMP", "configuredVrfs": ["default"], "activeVrfs": ["default"]}]}}],
"inputs": {"number": 1, "vrf": "MGMT"},
"expected": {"result": "failure", "messages": ["SNMP IPv4 ACL(s) not configured or active in vrf MGMT: ['ACL_IPV4_SNMP']"]},
"expected": {"result": "failure", "messages": ["VRF: MGMT - Following SNMP IPv4 ACL(s) not configured or active: ACL_IPV4_SNMP"]},
},
{
"name": "success",
Expand All @@ -78,14 +78,14 @@
"test": VerifySnmpIPv6Acl,
"eos_data": [{"ipv6AclList": {"aclList": []}}],
"inputs": {"number": 1, "vrf": "MGMT"},
"expected": {"result": "failure", "messages": ["Expected 1 SNMP IPv6 ACL(s) in vrf MGMT but got 0"]},
"expected": {"result": "failure", "messages": ["VRF: MGMT - Incorrect SNMP IPv6 ACL(s) - Expected: 1 Actual: 0"]},
},
{
"name": "failure-wrong-vrf",
"test": VerifySnmpIPv6Acl,
"eos_data": [{"ipv6AclList": {"aclList": [{"type": "Ip6Acl", "name": "ACL_IPV6_SNMP", "configuredVrfs": ["default"], "activeVrfs": ["default"]}]}}],
"inputs": {"number": 1, "vrf": "MGMT"},
"expected": {"result": "failure", "messages": ["SNMP IPv6 ACL(s) not configured or active in vrf MGMT: ['ACL_IPV6_SNMP']"]},
"expected": {"result": "failure", "messages": ["VRF: MGMT - Following SNMP IPv6 ACL(s) not configured or active: ACL_IPV6_SNMP"]},
},
{
"name": "success",
Expand All @@ -109,7 +109,7 @@
"inputs": {"location": "New York"},
"expected": {
"result": "failure",
"messages": ["Expected `New York` as the location, but found `Europe` instead."],
"messages": ["Incorrect SNMP location - Expected: New York Actual: Europe"],
},
},
{
Expand Down Expand Up @@ -148,7 +148,7 @@
"inputs": {"contact": "[email protected]"},
"expected": {
"result": "failure",
"messages": ["Expected `[email protected]` as the contact, but found `[email protected]` instead."],
"messages": ["Incorrect SNMP contact - Expected: [email protected] Actual: [email protected]"],
},
},
{
Expand Down Expand Up @@ -227,7 +227,7 @@
"inputs": {},
"expected": {
"result": "failure",
"messages": ["The following SNMP PDU counters are not found or have zero PDU counters:\n{'inGetPdus': 0, 'inSetPdus': 0}"],
"messages": ["The following SNMP PDU counters are not found or have zero PDU counters: inGetPdus, inSetPdus"],
},
},
{
Expand All @@ -245,7 +245,7 @@
"inputs": {"pdus": ["inGetPdus", "outTrapPdus"]},
"expected": {
"result": "failure",
"messages": ["The following SNMP PDU counters are not found or have zero PDU counters:\n{'inGetPdus': 'Not Found', 'outTrapPdus': 'Not Found'}"],
"messages": ["The following SNMP PDU counters are not found or have zero PDU counters: inGetPdus, outTrapPdus"],
},
},
{
Expand Down Expand Up @@ -319,9 +319,7 @@
"inputs": {},
"expected": {
"result": "failure",
"messages": [
"The following SNMP error counters are not found or have non-zero error counters:\n{'inVersionErrs': 1, 'inParseErrs': 2, 'outBadValueErrs': 2}"
],
"messages": ["The following SNMP error counters are not found or have non-zero error counters: inParseErrs, inVersionErrs, outBadValueErrs"],
},
},
{
Expand Down
Loading