Merge remote-tracking branch 'origin/master' into ab/fix-typing

# Conflicts:
#	CHANGES.rst
#	src/pytest_bdd/parser.py
#	src/pytest_bdd/scenario.py
#	src/pytest_bdd/utils.py

Author: youtux

CHANGES.rst

@@ -14,6 +14,7 @@ Added
 
 Changed
 +++++++
+* Step arguments ``"datatable"`` and ``"docstring"`` are now reserved, and they can't be used as step argument names.
 
 Deprecated
 ++++++++++
@@ -24,6 +25,7 @@ Removed
 Fixed
 +++++
 * Fixed an issue with the upcoming pytest release related to the use of ``@pytest.mark.usefixtures`` with an empty list.
+* Render template variables in docstrings and datatable cells with example table entries, as we already do for steps definitions.
 * Address many ``mypy`` warnings. The following private attributes are not available anymore (`#658 <https://github.com/pytest-dev/pytest-bdd/pull/658>`_):
   * ``_pytest.reports.TestReport.scenario`` (replaced by ``pytest_bdd.reporting.test_report_context`` WeakKeyDictionary)
   * ``__scenario__`` attribute of test functions generated by the ``@scenario`` (and ``@scenarios``) decorator (replaced by ``pytest_bdd.scenario.scenario_wrapper_template_registry`` WeakKeyDictionary)

README.rst

@@ -513,6 +513,58 @@ Example:
     def should_have_left_cucumbers(cucumbers, left):
         assert cucumbers["start"] - cucumbers["eat"] == left
 
+
+Example parameters from example tables can not only be used in steps, but also embedded directly within docstrings and datatables, allowing for dynamic substitution.
+This provides added flexibility for scenarios that require complex setups or validations.
+
+Example:
+
+.. code-block:: gherkin
+
+    # content of docstring_and_datatable_with_params.feature
+
+    Feature: Docstring and Datatable with example parameters
+        Scenario Outline: Using parameters in docstrings and datatables
+            Given the following configuration:
+                """
+                username: <username>
+                password: <password>
+                """
+            When the user logs in
+            Then the response should contain:
+                | field     | value      |
+                | username  | <username> |
+                | logged_in | true       |
+
+            Examples:
+            | username  | password  |
+            | user1     | pass123   |
+            | user2     | 123secure |
+
+.. code-block:: python
+
+    from pytest_bdd import scenarios, given, when, then
+    import json
+
+    # Load scenarios from the feature file
+    scenarios("docstring_and_datatable_with_params.feature")
+
+
+    @given("the following configuration:")
+    def given_user_config(docstring):
+        print(docstring)
+
+
+    @when("the user logs in")
+    def user_logs_in(logged_in):
+        logged_in = True
+
+
+    @then("the response should contain:")
+    def response_should_contain(datatable):
+        assert datatable[1][1] in ["user1", "user2"]
+
+
 Rules
 -----
 

src/pytest_bdd/exceptions.py

@@ -3,6 +3,10 @@
 from __future__ import annotations
 
 
+class StepImplementationError(Exception):
+    """Step implementation error."""
+
+
 class ScenarioIsDecoratorOnly(Exception):
     """Scenario can be only used as decorator."""
 

src/pytest_bdd/parser.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import copy
 import os.path
 import re
 import textwrap
@@ -19,7 +20,28 @@
 from .gherkin_parser import get_gherkin_document
 from .types import STEP_TYPE_BY_PARSER_KEYWORD
 
-STEP_PARAM_RE = re.compile(r"<(.+?)>")
+PARAM_RE = re.compile(r"<(.+?)>")
+
+
+def render_string(input_string: str, render_context: Mapping[str, object]) -> str:
+    """
+    Render the string with the given context,
+    but avoid replacing text inside angle brackets if context is missing.
+
+    Args:
+        input_string (str): The string for which to render/replace params.
+        render_context (Mapping[str, object]): The context for rendering the string.
+
+    Returns:
+        str: The rendered string with parameters replaced only if they exist in the context.
+    """
+
+    def replacer(m: re.Match) -> str:
+        varname = m.group(1)
+        # If the context contains the variable, replace it. Otherwise, leave it unchanged.
+        return str(render_context.get(varname, f"<{varname}>"))
+
+    return PARAM_RE.sub(replacer, input_string)
 
 
 def get_tag_names(tag_data: list[GherkinTag]) -> set[str]:
@@ -188,25 +210,25 @@ def render(self, context: Mapping[str, object]) -> Scenario:
         Returns:
             Scenario: A Scenario object with steps rendered based on the context.
         """
+        base_steps = self.all_background_steps + self._steps
         scenario_steps = [
             Step(
-                name=step.render(context),
+                name=render_string(step.name, context),
                 type=step.type,
                 indent=step.indent,
                 line_number=step.line_number,
                 keyword=step.keyword,
-                datatable=step.datatable,
-                docstring=step.docstring,
+                datatable=step.render_datatable(step.datatable, context) if step.datatable else None,
+                docstring=render_string(step.docstring, context) if step.docstring else None,
             )
-            for step in self._steps
+            for step in base_steps
         ]
-        steps = self.all_background_steps + scenario_steps
         return Scenario(
             feature=self.feature,
             keyword=self.keyword,
-            name=self.name,
+            name=render_string(self.name, context),
             line_number=self.line_number,
-            steps=steps,
+            steps=scenario_steps,
             tags=self.tags,
             description=self.description,
             rule=self.rule,
@@ -298,31 +320,24 @@ def __str__(self) -> str:
         """
         return f'{self.type.capitalize()} "{self.name}"'
 
-    @property
-    def params(self) -> tuple[str, ...]:
-        """Get the parameters in the step name.
-
-        Returns:
-            Tuple[str, ...]: A tuple of parameter names found in the step name.
+    @staticmethod
+    def render_datatable(datatable: DataTable, context: Mapping[str, object]) -> DataTable:
         """
-        return tuple(frozenset(STEP_PARAM_RE.findall(self.name)))
-
-    def render(self, context: Mapping[str, object]) -> str:
-        """Render the step name with the given context, but avoid replacing text inside angle brackets if context is missing.
+        Render the datatable with the given context,
+        but avoid replacing text inside angle brackets if context is missing.
 
         Args:
-            context (Mapping[str, object]): The context for rendering the step name.
+            datatable (DataTable): The datatable to render.
+            context (Mapping[str, object]): The context for rendering the datatable.
 
         Returns:
-            str: The rendered step name with parameters replaced only if they exist in the context.
+            datatable (DataTable): The rendered datatable with parameters replaced only if they exist in the context.
         """
-
-        def replacer(m: re.Match) -> str:
-            varname = m.group(1)
-            # If the context contains the variable, replace it. Otherwise, leave it unchanged.
-            return str(context.get(varname, f"<{varname}>"))
-
-        return STEP_PARAM_RE.sub(replacer, self.name)
+        rendered_datatable = copy.deepcopy(datatable)
+        for row in rendered_datatable.rows:
+            for cell in row.cells:
+                cell.value = render_string(cell.value, context)
+        return rendered_datatable
 
 
 @dataclass(eq=False)

src/pytest_bdd/scenario.py

@@ -18,6 +18,7 @@
 import os
 import re
 from collections.abc import Iterable, Iterator
+from inspect import signature
 from typing import TYPE_CHECKING, Any, Callable, TypeVar, cast
 from weakref import WeakKeyDictionary
 
@@ -28,7 +29,14 @@
 from .compat import getfixturedefs, inject_fixture
 from .feature import get_feature, get_features
 from .steps import StepFunctionContext, get_step_fixture_name, step_function_context_registry
-from .utils import CONFIG_STACK, get_args, get_caller_module_locals, get_caller_module_path, registry_get_safe
+from .utils import (
+    CONFIG_STACK,
+    get_caller_module_locals,
+    get_caller_module_path,
+    get_required_args,
+    identity,
+    registry_get_safe,
+)
 
 if TYPE_CHECKING:
     from _pytest.mark.structures import ParameterSet
@@ -40,10 +48,13 @@
 
 logger = logging.getLogger(__name__)
 
-
 PYTHON_REPLACE_REGEX = re.compile(r"\W")
 ALPHA_REGEX = re.compile(r"^\d+_*")
 
+STEP_ARGUMENT_DATATABLE = "datatable"
+STEP_ARGUMENT_DOCSTRING = "docstring"
+STEP_ARGUMENTS_RESERVED_NAMES = {STEP_ARGUMENT_DATATABLE, STEP_ARGUMENT_DOCSTRING}
+
 scenario_wrapper_template_registry: WeakKeyDictionary[Callable[..., object], ScenarioTemplate] = WeakKeyDictionary()
 
 
@@ -173,11 +184,35 @@ def get_step_function(request: FixtureRequest, step: Step) -> StepFunctionContex
             return None
 
 
+def parse_step_arguments(step: Step, context: StepFunctionContext) -> dict[str, object]:
+    """Parse step arguments."""
+    parsed_args = context.parser.parse_arguments(step.name)
+
+    assert parsed_args is not None, (
+        f"Unexpected `NoneType` returned from " f"parse_arguments(...) in parser: {context.parser!r}"
+    )
+
+    reserved_args = set(parsed_args.keys()) & STEP_ARGUMENTS_RESERVED_NAMES
+    if reserved_args:
+        reserved_arguments_str = ", ".join(repr(arg) for arg in reserved_args)
+        raise exceptions.StepImplementationError(
+            f"Step {step.name!r} defines argument names that are reserved: {reserved_arguments_str}. "
+            "Please use different names."
+        )
+
+    converted_args = {key: (context.converters.get(key, identity)(value)) for key, value in parsed_args.items()}
+
+    return converted_args
+
+
 def _execute_step_function(
     request: FixtureRequest, scenario: Scenario, step: Step, context: StepFunctionContext
 ) -> None:
     """Execute step function."""
     __tracebackhide__ = True
+
+    func_sig = signature(context.step_func)
+
     kw = {
         "request": request,
         "feature": scenario.feature,
@@ -186,38 +221,32 @@ def _execute_step_function(
         "step_func": context.step_func,
         "step_func_args": {},
     }
-
     request.config.hook.pytest_bdd_before_step(**kw)
 
-    # Get the step argument values.
-    converters = context.converters
-    kwargs = {}
-    args = get_args(context.step_func)
-
     try:
-        parsed_args = context.parser.parse_arguments(step.name)
-        assert parsed_args is not None, (
-            f"Unexpected `NoneType` returned from " f"parse_arguments(...) in parser: {context.parser!r}"
-        )
-
-        for arg, value in parsed_args.items():
-            if arg in converters:
-                value = converters[arg](value)
-            kwargs[arg] = value
+        parsed_args = parse_step_arguments(step=step, context=context)
 
-        if step.datatable is not None:
-            kwargs["datatable"] = step.datatable.raw()
+        # Filter out the arguments that are not in the function signature
+        kwargs = {k: v for k, v in parsed_args.items() if k in func_sig.parameters}
 
-        if step.docstring is not None:
-            kwargs["docstring"] = step.docstring
+        if STEP_ARGUMENT_DATATABLE in func_sig.parameters and step.datatable is not None:
+            kwargs[STEP_ARGUMENT_DATATABLE] = step.datatable.raw()
+        if STEP_ARGUMENT_DOCSTRING in func_sig.parameters and step.docstring is not None:
+            kwargs[STEP_ARGUMENT_DOCSTRING] = step.docstring
 
-        kwargs = {arg: kwargs[arg] if arg in kwargs else request.getfixturevalue(arg) for arg in args}
+        # Fill the missing arguments requesting the fixture values
+        kwargs |= {
+            arg: request.getfixturevalue(arg) for arg in get_required_args(context.step_func) if arg not in kwargs
+        }
 
         kw["step_func_args"] = kwargs
 
         request.config.hook.pytest_bdd_before_step_call(**kw)
-        # Execute the step as if it was a pytest fixture, so that we can allow "yield" statements in it
+
+        # Execute the step as if it was a pytest fixture using `call_fixture_func`,
+        # so that we can allow "yield" statements in it
         return_value = call_fixture_func(fixturefunc=context.step_func, request=request, kwargs=kwargs)
+
     except Exception as exception:
         request.config.hook.pytest_bdd_step_error(exception=exception, **kw)
         raise
@@ -270,7 +299,7 @@ def decorator(*args: Callable[..., T]) -> Callable[[FixtureRequest, dict[str, st
                 "scenario function can only be used as a decorator. Refer to the documentation."
             )
         [fn] = args
-        func_args = get_args(fn)
+        func_args = get_required_args(fn)
 
         def scenario_wrapper(request: FixtureRequest, _pytest_bdd_example: dict[str, str]) -> T:
             __tracebackhide__ = True

src/pytest_bdd/utils.py

@@ -21,13 +21,12 @@
 CONFIG_STACK: list[Config] = []
 
 
-def get_args(func: Callable[..., object]) -> list[str]:
-    """Get a list of argument names for a function.
+def get_required_args(func: Callable[..., object]) -> list[str]:
+    """Get a list of argument that are required for a function.
 
     :param func: The function to inspect.
 
     :return: A list of argument names.
-    :rtype: list
     """
     params = signature(func).parameters.values()
     return [
@@ -86,6 +85,11 @@ def setdefault(obj: object, name: str, default: T) -> T:
         return default
 
 
+def identity(x: T) -> T:
+    """Return the argument."""
+    return x
+
+
 @overload
 def registry_get_safe(registry: WeakKeyDictionary[K, V], key: object, default: T) -> V | T: ...
 @overload