Also give parameter attribution for other kinds of failures (#44)

Followup to #41 to handle errors in tests that are not assertion errors.

Author: s3cur3

lib/parameterized_test.ex

@@ -133,9 +133,9 @@ defmodule ParameterizedTest do
         test "#{full_test_name}", unquote(context_ast) do
           try do
             unquote(blocks)
-          rescue
-            e in ExUnit.AssertionError ->
-              ParameterizedTest.Backtrace.add_test_context(e, __STACKTRACE__, @param_test_context)
+          catch
+            category, reason ->
+              ParameterizedTest.Backtrace.add_test_context({category, reason}, __STACKTRACE__, @param_test_context)
           end
         end
       end
@@ -195,9 +195,9 @@ defmodule ParameterizedTest do
           feature "#{@full_test_name}", unquote(context_ast) do
             try do
               unquote(blocks)
-            rescue
-              e in ExUnit.AssertionError ->
-                ParameterizedTest.Backtrace.add_test_context(e, __STACKTRACE__, @param_test_context)
+            catch
+              category, reason ->
+                ParameterizedTest.Backtrace.add_test_context({category, reason}, __STACKTRACE__, @param_test_context)
             end
           end
         end

lib/parameterized_test/backtrace.ex

@@ -2,9 +2,17 @@ defmodule ParameterizedTest.Backtrace do
   @moduledoc false
   alias ParameterizedTest.Parser
 
-  @spec add_test_context(ExUnit.AssertionError.t(), [tuple()], Parser.context()) :: no_return()
-  def add_test_context(%ExUnit.AssertionError{} = e, bt, context) do
-    reraise e, augment_backtrace(bt, context)
+  @spec add_test_context({atom(), term()}, [tuple()], Parser.context()) :: no_return()
+  def add_test_context({:error, %{__exception__: true} = exception}, bt, context) do
+    reraise exception, augment_backtrace(bt, context)
+  end
+
+  def add_test_context({:error, payload}, bt, context) do
+    reraise ErlangError.normalize(payload, bt), augment_backtrace(bt, context)
+  end
+
+  def add_test_context({kind, payload}, bt, context) do
+    reraise RuntimeError.exception("#{kind}: #{inspect(payload)}"), augment_backtrace(bt, context)
   end
 
   defp augment_backtrace(bt, context) do

test/parameterized_test/backtrace_test.exs

@@ -21,10 +21,10 @@ defmodule ParameterizedTest.BacktraceTest do
 
       try do
         assert not should_fail?
-      rescue
-        e in ExUnit.AssertionError ->
+      catch
+        category, reason ->
           try do
-            ParameterizedTest.Backtrace.add_test_context(e, __STACKTRACE__, context)
+            ParameterizedTest.Backtrace.add_test_context({category, reason}, __STACKTRACE__, context)
           rescue
             ExUnit.AssertionError ->
               assert [failing_line, parameter_line | _] = __STACKTRACE__
@@ -44,6 +44,20 @@ defmodule ParameterizedTest.BacktraceTest do
     end
   end
 
+  test "translates Erlang errors" do
+    assert_raise ArithmeticError, fn ->
+      context = [file: __ENV__.file, min_line: __ENV__.line, raw: "| true         |"]
+      ParameterizedTest.Backtrace.add_test_context({:error, :badarith}, [], context)
+    end
+  end
+
+  test "turns other errors into RuntimeErrors" do
+    assert_raise RuntimeError, fn ->
+      context = [file: __ENV__.file, min_line: __ENV__.line, raw: "| true         |"]
+      ParameterizedTest.Backtrace.add_test_context({:timeout, {GenServer, :call, [self(), :slow_call, 0]}}, [], context)
+    end
+  end
+
   @tag failure_with_backtrace: true
   param_test "points to line #{__ENV__.line + 4} when a test fails",
              """
@@ -149,4 +163,83 @@ defmodule ParameterizedTest.BacktraceTest do
   } do
     assert not gets_free_shipping?
   end
+
+  @tag failure_with_backtrace: true
+  param_test "handles other exceptions, attribute to line #{__ENV__.line + 4}",
+             """
+             | should_fail? |
+             | false        |
+             | true         |
+             """,
+             %{should_fail?: should_fail?} do
+    if should_fail? do
+      raise "test failed"
+    else
+      assert 1 == 1
+    end
+  end
+
+  @tag failure_with_backtrace: true
+  param_test "handles code errors, attribute to line #{__ENV__.line + 4}",
+             """
+             | should_fail? |
+             | false        |
+             | true         |
+             """,
+             %{should_fail?: should_fail?} do
+    if should_fail? do
+      assert Code.eval_string("nil + 1") == 2
+    else
+      assert 1 == 1
+    end
+  end
+
+  defmodule SlowGenServer do
+    @moduledoc false
+    @behaviour GenServer
+
+    def start_link(opts \\ []), do: GenServer.start_link(__MODULE__, opts, name: __MODULE__)
+
+    @impl GenServer
+    def init(_), do: {:ok, []}
+
+    def slow_call(pid) do
+      GenServer.call(pid, :slow_call, 0)
+    end
+
+    @impl GenServer
+    def handle_call(:slow_call, _from, state) do
+      :timer.sleep(1000)
+      {:reply, :ok, state}
+    end
+  end
+
+  @tag failure_with_backtrace: true
+  param_test "handles non-assertion errors too, attribute to line #{__ENV__.line + 4}",
+             """
+             | should_fail? |
+             | false        |
+             | true         |
+             """,
+             %{should_fail?: should_fail?} do
+    if should_fail? do
+      {:ok, pid} = SlowGenServer.start_link()
+      SlowGenServer.slow_call(pid)
+    end
+  end
+
+  @tag skip: true
+  @tag failure_with_backtrace: true
+  param_feature "handles non-assertion errors in features, attribute to line #{__ENV__.line + 4}",
+                """
+                | should_fail? |
+                | false        |
+                | true         |
+                """,
+                %{should_fail?: should_fail?} do
+    if should_fail? do
+      {:ok, pid} = SlowGenServer.start_link()
+      SlowGenServer.slow_call(pid)
+    end
+  end
 end

test/parameterized_test_test.exs

@@ -354,6 +354,17 @@ defmodule ParameterizedTestTest do
              """ do
     flunk("This test should not run")
   end
+
+  param_test "applies param_test: true to all parameterized tests",
+             """
+             | text     | url                  |
+             |----------|----------------------|
+             | "GitHub" | "https://github.com" |
+             | "Google" | "https://google.com" |
+             """,
+             context do
+    assert context[:param_test] == true
+  end
 end
 
 defmodule ParameterizedTestTest.WallabyTest do
@@ -373,4 +384,15 @@ defmodule ParameterizedTestTest.WallabyTest do
     |> visit(url)
     |> assert_has(Wallaby.Query.text(text, minimum: 1))
   end
+
+  param_feature "applies param_feature: true to all parameterized tests",
+                """
+                | text     | url                  |
+                |----------|----------------------|
+                | "GitHub" | "https://github.com" |
+                | "Google" | "https://google.com" |
+                """,
+                context do
+    assert context[:param_test] == true
+  end
 end