Conformance tests: Fix manual scoring discrepancy for pyre (#1770)

Pyre infers "Literal[1]" instead of "int", which seems legal. Change
the test to get an int from a function parameter instead, so that all
type checkers are aligned.

Part of #1692.

Author: JelleZijlstra

conformance/results/mypy/generics_paramspec_semantics.toml

@@ -4,13 +4,13 @@ generics_paramspec_semantics.py:26: error: Unexpected keyword argument "a"  [cal
 generics_paramspec_semantics.py:26: error: Unexpected keyword argument "b"  [call-arg]
 generics_paramspec_semantics.py:27: error: Argument 2 has incompatible type "str"; expected "bool"  [arg-type]
 generics_paramspec_semantics.py:61: error: Argument 2 to "func1" has incompatible type "Callable[[NamedArg(int, 'y')], int]"; expected "Callable[[NamedArg(int, 'x')], int]"  [arg-type]
-generics_paramspec_semantics.py:97: error: Argument 1 has incompatible type "int"; expected "str"  [arg-type]
-generics_paramspec_semantics.py:107: error: Argument 1 has incompatible type "int"; expected "bool"  [arg-type]
-generics_paramspec_semantics.py:119: error: Argument 1 has incompatible type "int"; expected "str"  [arg-type]
-generics_paramspec_semantics.py:126: error: Argument 1 to "expects_int_first" has incompatible type "Callable[[str], int]"; expected "Callable[[int], int]"  [arg-type]
-generics_paramspec_semantics.py:126: note: This is likely because "one" has named arguments: "x". Consider marking them positional-only
-generics_paramspec_semantics.py:131: error: Argument 1 to "expects_int_first" has incompatible type "Callable[[NamedArg(int, 'x')], int]"; expected "Callable[[int, NamedArg(int, 'x')], int]"  [arg-type]
-generics_paramspec_semantics.py:136: error: Argument 1 to "expects_int_first" has incompatible type "Callable[[KwArg(int)], int]"; expected "Callable[[int, KwArg(int)], int]"  [arg-type]
+generics_paramspec_semantics.py:98: error: Argument 1 has incompatible type "int"; expected "str"  [arg-type]
+generics_paramspec_semantics.py:108: error: Argument 1 has incompatible type "int"; expected "bool"  [arg-type]
+generics_paramspec_semantics.py:120: error: Argument 1 has incompatible type "int"; expected "str"  [arg-type]
+generics_paramspec_semantics.py:127: error: Argument 1 to "expects_int_first" has incompatible type "Callable[[str], int]"; expected "Callable[[int], int]"  [arg-type]
+generics_paramspec_semantics.py:127: note: This is likely because "one" has named arguments: "x". Consider marking them positional-only
+generics_paramspec_semantics.py:132: error: Argument 1 to "expects_int_first" has incompatible type "Callable[[NamedArg(int, 'x')], int]"; expected "Callable[[int, NamedArg(int, 'x')], int]"  [arg-type]
+generics_paramspec_semantics.py:137: error: Argument 1 to "expects_int_first" has incompatible type "Callable[[KwArg(int)], int]"; expected "Callable[[int, KwArg(int)], int]"  [arg-type]
 """
 conformance_automated = "Pass"
 errors_diff = """

conformance/results/mypy/version.toml

@@ -1,2 +1,2 @@
 version = "mypy 1.10.0"
-test_duration = 1.5
+test_duration = 1.1

conformance/results/pyre/generics_paramspec_semantics.toml

@@ -4,15 +4,13 @@ generics_paramspec_semantics.py:26:0 Unexpected keyword [28]: Unexpected keyword
 generics_paramspec_semantics.py:27:8 Incompatible parameter type [6]: In anonymous call, for 2nd positional argument, expected `bool` but got `str`.
 generics_paramspec_semantics.py:46:16 Incompatible parameter type [6]: In call `func1`, for 2nd positional argument, expected `typing.Callable[generics_paramspec_semantics.P, int]` but got `typing.Callable(y_x)[[Named(y, int), Named(x, str)], int]`.
 generics_paramspec_semantics.py:61:22 Incompatible parameter type [6]: In call `func1`, for 2nd positional argument, expected `typing.Callable[generics_paramspec_semantics.P, int]` but got `typing.Callable(keyword_only_y)[[KeywordOnly(y, int)], int]`.
-generics_paramspec_semantics.py:81:0 Incompatible parameter type [6]: In call `assert_type`, for 1st positional argument, expected `Y[int, [int]]` but got `Y[typing_extensions.Literal[1], [int]]`.
-generics_paramspec_semantics.py:97:3 Incompatible parameter type [6]: In anonymous call, for 1st positional argument, expected `str` but got `int`.
-generics_paramspec_semantics.py:107:3 Incompatible parameter type [6]: In anonymous call, for 1st positional argument, expected `bool` but got `int`.
-generics_paramspec_semantics.py:119:3 Incompatible parameter type [6]: In anonymous call, for 1st positional argument, expected `str` but got `int`.
-generics_paramspec_semantics.py:126:1 Invalid decoration [56]: Pyre doesn't yet support decorators with ParamSpec applied to generic functions Please add # pyre-ignore[56] to `generics_paramspec_semantics.expects_int_first`.
-generics_paramspec_semantics.py:131:1 Invalid decoration [56]: Pyre doesn't yet support decorators with ParamSpec applied to generic functions Please add # pyre-ignore[56] to `generics_paramspec_semantics.expects_int_first`.
-generics_paramspec_semantics.py:136:1 Invalid decoration [56]: Pyre doesn't yet support decorators with ParamSpec applied to generic functions Please add # pyre-ignore[56] to `generics_paramspec_semantics.expects_int_first`.
+generics_paramspec_semantics.py:98:3 Incompatible parameter type [6]: In anonymous call, for 1st positional argument, expected `str` but got `int`.
+generics_paramspec_semantics.py:108:3 Incompatible parameter type [6]: In anonymous call, for 1st positional argument, expected `bool` but got `int`.
+generics_paramspec_semantics.py:120:3 Incompatible parameter type [6]: In anonymous call, for 1st positional argument, expected `str` but got `int`.
+generics_paramspec_semantics.py:127:1 Invalid decoration [56]: Pyre doesn't yet support decorators with ParamSpec applied to generic functions Please add # pyre-ignore[56] to `generics_paramspec_semantics.expects_int_first`.
+generics_paramspec_semantics.py:132:1 Invalid decoration [56]: Pyre doesn't yet support decorators with ParamSpec applied to generic functions Please add # pyre-ignore[56] to `generics_paramspec_semantics.expects_int_first`.
+generics_paramspec_semantics.py:137:1 Invalid decoration [56]: Pyre doesn't yet support decorators with ParamSpec applied to generic functions Please add # pyre-ignore[56] to `generics_paramspec_semantics.expects_int_first`.
 """
-conformance_automated = "Fail"
+conformance_automated = "Pass"
 errors_diff = """
-Line 81: Unexpected errors ['generics_paramspec_semantics.py:81:0 Incompatible parameter type [6]: In call `assert_type`, for 1st positional argument, expected `Y[int, [int]]` but got `Y[typing_extensions.Literal[1], [int]]`.']
 """

conformance/results/pyre/version.toml

@@ -1,2 +1,2 @@
 version = "pyre 0.9.21"
-test_duration = 3.6
+test_duration = 1.7

conformance/results/pyright/generics_paramspec_semantics.toml

@@ -14,20 +14,20 @@ generics_paramspec_semantics.py:61:23 - error: Argument of type "(*, y: int) ->
   Type "(*, y: int) -> int" is incompatible with type "(*, x: int) -> int"
     Extra parameter "y"
     Missing keyword parameter "x" (reportArgumentType)
-generics_paramspec_semantics.py:97:4 - error: Argument of type "Literal[1]" cannot be assigned to parameter of type "str"
+generics_paramspec_semantics.py:98:4 - error: Argument of type "Literal[1]" cannot be assigned to parameter of type "str"
   "Literal[1]" is incompatible with "str" (reportArgumentType)
-generics_paramspec_semantics.py:107:4 - error: Argument of type "Literal[1]" cannot be assigned to parameter "args" of type "bool"
+generics_paramspec_semantics.py:108:4 - error: Argument of type "Literal[1]" cannot be assigned to parameter "args" of type "bool"
   "Literal[1]" is incompatible with "bool" (reportArgumentType)
-generics_paramspec_semantics.py:119:4 - error: Argument of type "Literal[1]" cannot be assigned to parameter of type "str"
+generics_paramspec_semantics.py:120:4 - error: Argument of type "Literal[1]" cannot be assigned to parameter of type "str"
   "Literal[1]" is incompatible with "str" (reportArgumentType)
-generics_paramspec_semantics.py:126:2 - error: Argument of type "(x: str) -> int" cannot be assigned to parameter "x" of type "(int, **P@expects_int_first) -> int" in function "expects_int_first"
+generics_paramspec_semantics.py:127:2 - error: Argument of type "(x: str) -> int" cannot be assigned to parameter "x" of type "(int, **P@expects_int_first) -> int" in function "expects_int_first"
   Type "(x: str) -> int" is incompatible with type "(int, **P@expects_int_first) -> int"
     Parameter 1: type "int" is incompatible with type "str"
       "int" is incompatible with "str" (reportArgumentType)
-generics_paramspec_semantics.py:131:2 - error: Argument of type "(*, x: int) -> int" cannot be assigned to parameter "x" of type "(int, **P@expects_int_first) -> int" in function "expects_int_first"
+generics_paramspec_semantics.py:132:2 - error: Argument of type "(*, x: int) -> int" cannot be assigned to parameter "x" of type "(int, **P@expects_int_first) -> int" in function "expects_int_first"
   Type "(*, x: int) -> int" is incompatible with type "(int, **P@expects_int_first) -> int"
     Function accepts too many positional parameters; expected 0 but received 1 (reportArgumentType)
-generics_paramspec_semantics.py:136:2 - error: Argument of type "(**kwargs: int) -> int" cannot be assigned to parameter "x" of type "(int, **P@expects_int_first) -> int" in function "expects_int_first"
+generics_paramspec_semantics.py:137:2 - error: Argument of type "(**kwargs: int) -> int" cannot be assigned to parameter "x" of type "(int, **P@expects_int_first) -> int" in function "expects_int_first"
   Type "(**kwargs: int) -> int" is incompatible with type "(int, **P@expects_int_first) -> int"
     Function accepts too many positional parameters; expected 0 but received 1 (reportArgumentType)
 """

conformance/results/pyright/version.toml

@@ -1,2 +1,2 @@
 version = "pyright 1.1.366"
-test_duration = 1.4
+test_duration = 1.3

conformance/results/pytype/generics_paramspec_semantics.toml

@@ -10,10 +10,10 @@ errors_diff = """
 Line 26: Expected 1 errors
 Line 27: Expected 1 errors
 Line 61: Expected 1 errors
-Line 97: Expected 1 errors
-Line 107: Expected 1 errors
-Line 119: Expected 1 errors
-Line 126: Expected 1 errors
-Line 131: Expected 1 errors
-Line 136: Expected 1 errors
+Line 98: Expected 1 errors
+Line 108: Expected 1 errors
+Line 120: Expected 1 errors
+Line 127: Expected 1 errors
+Line 132: Expected 1 errors
+Line 137: Expected 1 errors
 """

conformance/results/pytype/version.toml

@@ -1,2 +1,2 @@
 version = "pytype 2024.04.11"
-test_duration = 32.9
+test_duration = 27.9

conformance/results/results.html

@@ -159,16 +159,16 @@ <h3>Python Type System Conformance Test Results</h3>
         <div class="table_container"><table><tbody>
 <tr><th class="col1">&nbsp;</th>
 <th class='tc-header'><div class='tc-name'>mypy 1.10.0</div>
-<div class='tc-time'>1.5sec</div>
+<div class='tc-time'>1.1sec</div>
 </th>
 <th class='tc-header'><div class='tc-name'>pyright 1.1.366</div>
-<div class='tc-time'>1.4sec</div>
+<div class='tc-time'>1.3sec</div>
 </th>
 <th class='tc-header'><div class='tc-name'>pyre 0.9.21</div>
-<div class='tc-time'>3.6sec</div>
+<div class='tc-time'>1.7sec</div>
 </th>
 <th class='tc-header'><div class='tc-name'>pytype 2024.04.11</div>
-<div class='tc-time'>32.9sec</div>
+<div class='tc-time'>27.9sec</div>
 </th>
 </tr>
 <tr><th class="column" colspan="5">

conformance/tests/generics_paramspec_semantics.py

@@ -77,10 +77,11 @@ def callback_a(q: int, /) -> str:
     ...
 
 
-y1 = Y(callback_a, 1)
-assert_type(y1, Y[int, [int]])
-y2 = y1.f
-assert_type(y2, Callable[[int], str])
+def func(x: int) -> None:
+    y1 = Y(callback_a, x)
+    assert_type(y1, Y[int, [int]])
+    y2 = y1.f
+    assert_type(y2, Callable[[int], str])
 
 
 def bar(x: int, *args: bool) -> int: