separate protocol check in custom json formatter

Author: Ghesselink

features/environment.py

@@ -42,17 +42,9 @@ def before_feature(context, feature):
             }
         protocol_errors = protocol.enforce(convention_attrs)
         for error in protocol_errors:
-            validation_outcome = ValidationOutcome(
-            outcome_code=ValidationOutcomeCode.EXECUTED,
-            observed=error,
-            expected=error,
-            feature=context.feature.name,
-            feature_version=1,
-            severity=OutcomeSeverity.ERROR,
-        )
-            context.protocol_errors.append(validation_outcome)
-
-    context.gherkin_outcomes = context.protocol_errors
+            context.protocol_errors.append(error)
+
+    context.gherkin_outcomes = []
 
 
 def before_scenario(context, scenario):
@@ -130,4 +122,8 @@ def get_or_create_instance_when_set(spf_id):
         outcomes_json_str = json.dumps(outcomes) #ncodes to utf-8 
         outcomes_bytes = outcomes_json_str.encode("utf-8") 
         for formatter in filter(lambda f: hasattr(f, "embedding"), context._runner.formatters):
-            formatter.embedding(mime_type="application/json", data=outcomes_bytes, target='feature', attribute_name='validation_outcomes')
+            formatter.embedding(mime_type="application/json", data=outcomes_bytes, target='feature', attribute_name='validation_outcomes')
+
+            # embed protocol errors
+            protocol_errors_bytes = json.dumps(context.protocol_errors).encode("utf-8")
+            formatter.embedding(mime_type="application/json", data=protocol_errors_bytes, target='feature', attribute_name='protocol_errors') 

main.py

@@ -128,6 +128,11 @@ def run(filename, rule_type=RuleType.ALL, with_console_output=False, execution_m
                 except KeyError:
                     el_list = []
                 for el in el_list:
+                    protocol_errors = json.loads(base64.b64decode(el.get('protocol_errors', [{}])[0].get('data', '')).decode('utf-8')) if el.get('protocol_errors') else []
+                    if protocol_errors:
+                        yield {
+                            'protocol_errors': protocol_errors,
+                        }
                     scenario_validation_outcomes = json.loads(base64.b64decode(el.get('validation_outcomes', [{}])[0].get('data', '')).decode('utf-8')) if el.get('validation_outcomes') else []
                     scenario_info = {
                         'scenario_name': el['name'],

test/test_main.py

@@ -50,7 +50,9 @@ def test_invocation(filename):
         print('The Gherkin tests did not run for the specified test file, and the JSON report is empty. Please review the test file for any errors.')
 
     rule_is_disabled = feature_info['rule_is_disabled']
-    validation_outcomes = gherkin_results[1:]
+    protocol_errors = next((d for d in gherkin_results if 'protocol_errors' in d), None)
+
+    validation_outcomes = [d for d in gherkin_results[1:] if 'protocol_errors' not in d]
 
     error_outcomes = [outcome for outcome in validation_outcomes if outcome['severity'] in ['Error', 'Warning']]
     activating_outcomes = [outcome for outcome in validation_outcomes if outcome['severity'] == 'Executed']
@@ -75,6 +77,16 @@ def test_invocation(filename):
         # state originating from given steps. Therefore when results without disabled messages
         # is empty, it means that the rule has not been activated. I.e given statements
         # did not result in an actionable set of instances at the time of the first then step.
+        
+        #first, check if there are no protocol errors
+        if protocol_errors:
+            red_text = "\033[91m"
+            reset_text = "\033[0m"
+            print(f'{red_text}\n\nWARNING: The following protocol errors have been found:{reset_text}')
+            print(tabulate.tabulate([[error] for error in protocol_errors['protocol_errors']], headers=['Details'], tablefmt='fancy_grid'))
+            assert False # table should be printed before the assertion
+        
+        
         if base.startswith('fail'):
             assert len(error_outcomes) > 0
         elif base.startswith('pass'):