feat(types) Allow setting types for attributes (#5460)

* init

* add comment

* style: pre-commit fixes

* fix extra ;

* Add annotation helper for older python versions

* style: pre-commit fixes

* test writing __annotations__ to __dict__

* style: pre-commit fixes

* revert bac to __annotations__

* use getattr for automatic init

* use std::move

* update helper

* remove stdmove

* test isinstance

* style: pre-commit fixes

* add #if 3.9

* style: pre-commit fixes

* use getattr

* add dir

* use hasattr

* try setattr

* add c++17 guard

* style: pre-commit fixes

* add compile guard

* style: pre-commit fixes

* cleanup

* comments and function name cleanup

* style: pre-commit fixes

* update test case

* style: pre-commit fixes

* add write

* added instance check

* style: pre-commit fixes

* Test for `__cpp_inline_variables` and use `static_assert()`

* Add guard: __annotations__["list_int"] was set already.

* Avoid explicit `false` in `static_assert()`.

* add redeclaration test

* style: pre-commit fixes

* add handle support to attr_with_type_hint

* store reintpreted_key

* style: pre-commit fixes

* fix str namespace

* fix scope?

* string wrap

* clang tidy

* Swap order of attr_with_type_hint implementation in cast.h (so that the `const char *` overload appears first).

* Reuse `const char *` overload from `handle` overload.

* Added comment

* style: pre-commit fixes

* line up comment

* fixed spelling error

* Add missing period

* Introduce `detail::always_false<>` to make the new `static_assert()` more readable.

* Copy `attr` comment for `const char *` overload, for consistency.

* static_assert(std::false_type::value, ...)

* Revert "static_assert(std::false_type::value, ...)"

This reverts commit 999b668.

* Add comment for `always_false`

* add test for inspect.get_annotations()

* style: pre-commit fixes

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Ralf W. Grosse-Kunstleve <[email protected]>

Author: 3 people

include/pybind11/cast.h

@@ -1366,6 +1366,31 @@ object object_or_cast(T &&o) {
     return pybind11::cast(std::forward<T>(o));
 }
 
+// Declared in pytypes.h:
+// Implemented here so that make_caster<T> can be used.
+template <typename D>
+template <typename T>
+str_attr_accessor object_api<D>::attr_with_type_hint(const char *key) const {
+#if !defined(__cpp_inline_variables)
+    static_assert(always_false<T>::value,
+                  "C++17 feature __cpp_inline_variables not available: "
+                  "https://en.cppreference.com/w/cpp/language/static#Static_data_members");
+#endif
+    object ann = annotations();
+    if (ann.contains(key)) {
+        throw std::runtime_error("__annotations__[\"" + std::string(key) + "\"] was set already.");
+    }
+    ann[key] = make_caster<T>::name.text;
+    return {derived(), key};
+}
+
+template <typename D>
+template <typename T>
+obj_attr_accessor object_api<D>::attr_with_type_hint(handle key) const {
+    (void) attr_with_type_hint<T>(key.cast<std::string>().c_str());
+    return {derived(), reinterpret_borrow<object>(key)};
+}
+
 // Placeholder type for the unneeded (and dead code) static variable in the
 // PYBIND11_OVERRIDE_OVERRIDE macro
 struct override_unused {};

include/pybind11/detail/common.h

@@ -627,6 +627,14 @@ struct instance {
 static_assert(std::is_standard_layout<instance>::value,
               "Internal error: `pybind11::detail::instance` is not standard layout!");
 
+// Some older compilers (e.g. gcc 9.4.0) require
+//     static_assert(always_false<T>::value, "...");
+// instead of
+//     static_assert(false, "...");
+// to trigger the static_assert() in a template only if it is actually instantiated.
+template <typename>
+struct always_false : std::false_type {};
+
 /// from __cpp_future__ import (convenient aliases from C++14/17)
 #if defined(PYBIND11_CPP14)
 using std::conditional_t;

include/pybind11/pytypes.h

@@ -113,6 +113,17 @@ class object_api : public pyobject_tag {
     /// See above (the only difference is that the key is provided as a string literal)
     str_attr_accessor attr(const char *key) const;
 
+    /** \rst
+         Similar to the above attr functions with the difference that the templated Type
+         is used to set the `__annotations__` dict value to the corresponding key. Worth noting
+         that attr_with_type_hint is implemented in cast.h.
+    \endrst */
+    template <typename T>
+    obj_attr_accessor attr_with_type_hint(handle key) const;
+    /// See above (the only difference is that the key is provided as a string literal)
+    template <typename T>
+    str_attr_accessor attr_with_type_hint(const char *key) const;
+
     /** \rst
         Matches * unpacking in Python, e.g. to unpack arguments out of a ``tuple``
         or ``list`` for a function call. Applying another * to the result yields
@@ -182,6 +193,9 @@ class object_api : public pyobject_tag {
     /// Get or set the object's docstring, i.e. ``obj.__doc__``.
     str_attr_accessor doc() const;
 
+    /// Get or set the object's annotations, i.e. ``obj.__annotations__``.
+    object annotations() const;
+
     /// Return the object's current reference count
     ssize_t ref_count() const {
 #ifdef PYPY_VERSION
@@ -2558,6 +2572,19 @@ str_attr_accessor object_api<D>::doc() const {
     return attr("__doc__");
 }
 
+template <typename D>
+object object_api<D>::annotations() const {
+#if PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION <= 9
+    // https://docs.python.org/3/howto/annotations.html#accessing-the-annotations-dict-of-an-object-in-python-3-9-and-older
+    if (!hasattr(derived(), "__annotations__")) {
+        setattr(derived(), "__annotations__", dict());
+    }
+    return attr("__annotations__");
+#else
+    return getattr(derived(), "__annotations__", dict());
+#endif
+}
+
 template <typename D>
 handle object_api<D>::get_type() const {
     return type::handle_of(derived());

include/pybind11/typing.h

@@ -82,6 +82,18 @@ class Optional : public object {
     using object::object;
 };
 
+template <typename T>
+class Final : public object {
+    PYBIND11_OBJECT_DEFAULT(Final, object, PyObject_Type)
+    using object::object;
+};
+
+template <typename T>
+class ClassVar : public object {
+    PYBIND11_OBJECT_DEFAULT(ClassVar, object, PyObject_Type)
+    using object::object;
+};
+
 template <typename T>
 class TypeGuard : public bool_ {
     using bool_::bool_;
@@ -251,6 +263,16 @@ struct handle_type_name<typing::Optional<T>> {
         = const_name("Optional[") + as_return_type<make_caster<T>>::name + const_name("]");
 };
 
+template <typename T>
+struct handle_type_name<typing::Final<T>> {
+    static constexpr auto name = const_name("Final[") + make_caster<T>::name + const_name("]");
+};
+
+template <typename T>
+struct handle_type_name<typing::ClassVar<T>> {
+    static constexpr auto name = const_name("ClassVar[") + make_caster<T>::name + const_name("]");
+};
+
 // TypeGuard and TypeIs use as_return_type to use the return type if available, which is usually
 // the narrower type.
 

tests/test_pytypes.cpp

@@ -1037,6 +1037,38 @@ TEST_SUBMODULE(pytypes, m) {
 #else
     m.attr("defined_PYBIND11_TEST_PYTYPES_HAS_RANGES") = false;
 #endif
+
+#if defined(__cpp_inline_variables)
+    // Exercises const char* overload:
+    m.attr_with_type_hint<py::typing::List<int>>("list_int") = py::list();
+    // Exercises py::handle overload:
+    m.attr_with_type_hint<py::typing::Set<py::str>>(py::str("set_str")) = py::set();
+
+    struct Empty {};
+    py::class_<Empty>(m, "EmptyAnnotationClass");
+
+    struct Static {};
+    auto static_class = py::class_<Static>(m, "Static");
+    static_class.def(py::init());
+    static_class.attr_with_type_hint<py::typing::ClassVar<float>>("x") = 1.0;
+    static_class.attr_with_type_hint<py::typing::ClassVar<py::typing::Dict<py::str, int>>>(
+        "dict_str_int")
+        = py::dict();
+
+    struct Instance {};
+    auto instance = py::class_<Instance>(m, "Instance", py::dynamic_attr());
+    instance.def(py::init());
+    instance.attr_with_type_hint<float>("y");
+
+    m.def("attr_with_type_hint_float_x",
+          [](py::handle obj) { obj.attr_with_type_hint<float>("x"); });
+
+    m.attr_with_type_hint<py::typing::Final<int>>("CONST_INT") = 3;
+
+    m.attr("defined___cpp_inline_variables") = true;
+#else
+    m.attr("defined___cpp_inline_variables") = false;
+#endif
     m.def("half_of_number", [](const RealNumber &x) { return RealNumber{x.value / 2}; });
     // std::vector<T>
     m.def("half_of_number_vector", [](const std::vector<RealNumber> &x) {

tests/test_pytypes.py

@@ -1103,6 +1103,96 @@ def test_dict_ranges(tested_dict, expected):
     assert m.transform_dict_plus_one(tested_dict) == expected
 
 
+# https://docs.python.org/3/howto/annotations.html#accessing-the-annotations-dict-of-an-object-in-python-3-9-and-older
+def get_annotations_helper(o):
+    if isinstance(o, type):
+        return o.__dict__.get("__annotations__", None)
+    return getattr(o, "__annotations__", None)
+
+
+@pytest.mark.skipif(
+    not m.defined___cpp_inline_variables,
+    reason="C++17 feature __cpp_inline_variables not available.",
+)
+def test_module_attribute_types() -> None:
+    module_annotations = get_annotations_helper(m)
+
+    assert module_annotations["list_int"] == "list[int]"
+    assert module_annotations["set_str"] == "set[str]"
+
+
+@pytest.mark.skipif(
+    not m.defined___cpp_inline_variables,
+    reason="C++17 feature __cpp_inline_variables not available.",
+)
+@pytest.mark.skipif(
+    sys.version_info < (3, 10),
+    reason="get_annotations function does not exist until Python3.10",
+)
+def test_get_annotations_compliance() -> None:
+    from inspect import get_annotations
+
+    module_annotations = get_annotations(m)
+
+    assert module_annotations["list_int"] == "list[int]"
+    assert module_annotations["set_str"] == "set[str]"
+
+
+@pytest.mark.skipif(
+    not m.defined___cpp_inline_variables,
+    reason="C++17 feature __cpp_inline_variables not available.",
+)
+def test_class_attribute_types() -> None:
+    empty_annotations = get_annotations_helper(m.EmptyAnnotationClass)
+    static_annotations = get_annotations_helper(m.Static)
+    instance_annotations = get_annotations_helper(m.Instance)
+
+    assert empty_annotations is None
+    assert static_annotations["x"] == "ClassVar[float]"
+    assert static_annotations["dict_str_int"] == "ClassVar[dict[str, int]]"
+
+    assert m.Static.x == 1.0
+
+    m.Static.x = 3.0
+    static = m.Static()
+    assert static.x == 3.0
+
+    static.dict_str_int["hi"] = 3
+    assert m.Static().dict_str_int == {"hi": 3}
+
+    assert instance_annotations["y"] == "float"
+    instance1 = m.Instance()
+    instance1.y = 4.0
+
+    instance2 = m.Instance()
+    instance2.y = 5.0
+
+    assert instance1.y != instance2.y
+
+
+@pytest.mark.skipif(
+    not m.defined___cpp_inline_variables,
+    reason="C++17 feature __cpp_inline_variables not available.",
+)
+def test_redeclaration_attr_with_type_hint() -> None:
+    obj = m.Instance()
+    m.attr_with_type_hint_float_x(obj)
+    assert get_annotations_helper(obj)["x"] == "float"
+    with pytest.raises(
+        RuntimeError, match=r'^__annotations__\["x"\] was set already\.$'
+    ):
+        m.attr_with_type_hint_float_x(obj)
+
+
+@pytest.mark.skipif(
+    not m.defined___cpp_inline_variables,
+    reason="C++17 feature __cpp_inline_variables not available.",
+)
+def test_final_annotation() -> None:
+    module_annotations = get_annotations_helper(m)
+    assert module_annotations["CONST_INT"] == "Final[int]"
+
+
 def test_arg_return_type_hints(doc):
     assert doc(m.half_of_number) == "half_of_number(arg0: Union[float, int]) -> float"
     assert m.half_of_number(2.0) == 1.0