Enabling coverage affects `@inferred` on 1.12

[mhauru]

We have a case over in TuringLang/DynamicPPL.jl#1099 where whether an @inferred test passes or fails depends on whether we call Pkg.test with or without coverage. This is new on 1.12. Feel free to move this issue if you think this is a Julia bug. I'm just utterly confused as to what coverage has to do with type inference.

Here's a minimal reproduction. I have the following package:

├── Project.toml
├── src
│   └── BugMWE.jl
└── test
    ├── Project.toml
    └── runtests.jl

The main Project.toml and BugMWE.jl are empty. test/Project.toml reads

[deps]
AbstractPPL = "7a57a42e-76ec-4ea3-a279-07e840d6d9cf"
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

and test/runtests.jl is

using AbstractPPL: VarName, getsym, @varname
using Test

VarNameTuple = NTuple{N,VarName} where {N}

@generated function unique_syms(::T) where {T<:VarNameTuple}
    retval = Expr(:tuple)
    syms = [first(vn.parameters) for vn in T.parameters]
    for sym in unique(syms)
        push!(retval.args, QuoteNode(sym))
    end
    return retval
end

function group_varnames_by_symbol(vns::VarNameTuple)
    syms = unique_syms(vns)
    elements = map(collect, tuple((filter(vn -> getsym(vn) == s, vns) for s in syms)...))
    return NamedTuple{syms}(elements)
end

@testset begin
    vns_tuple = (
        @varname(x), @varname(y[1]), @varname(x.a), @varname(z[15]), @varname(y[2])
    )
    @inferred group_varnames_by_symbol(vns_tuple)
end

For this project, julia --project=. -e 'import Pkg; Pkg.test(; coverage=true)' passes fine, but julia --project=. -e 'import Pkg; Pkg.test(; coverage=false)' gives

test set: Error During Test at /private/tmp/mwe/BugMWE/test/runtests.jl:21
  Got exception outside of a @test
  return type @NamedTuple{x::Vector{VarName{:x}}, y::Vector{VarName{:y, Accessors.IndexLens{Tuple{Int64}}}}, z::Vector{VarName{:z, Accessors.IndexLens{Tuple{Int64}}}}} does not match inferred return type NamedTuple{(:x, :y, :z)}
  Stacktrace:
   [1] error(s::String)
     @ Base ./error.jl:44
   [2] top-level scope
     @ /private/tmp/mwe/BugMWE/test/runtests.jl:22
   [3] macro expansion
     @ ~/.julia/juliaup/julia-1.12.1+0.aarch64.apple.darwin14/share/julia/stdlib/v1.12/Test/src/Test.jl:1776 [inlined]
   [4] macro expansion
     @ /private/tmp/mwe/BugMWE/test/runtests.jl:25 [inlined]
   [5] include(mapexpr::Function, mod::Module, _path::String)
     @ Base ./Base.jl:307
   [6] top-level scope
     @ none:6
   [7] eval(m::Module, e::Any)
     @ Core ./boot.jl:489
   [8] exec_options(opts::Base.JLOptions)
     @ Base ./client.jl:283
   [9] _start()
     @ Base ./client.jl:550
Test Summary: | Error  Total  Time
test set      |     1      1  1.9s

The problem is new on 1.12, on 1.11 the test passes regardless of coverage.

Julia version:

julia> versioninfo()
Julia Version 1.12.1
Commit ba1e628ee49 (2025-10-17 13:02 UTC)
Build Info:
  Official https://julialang.org release
Platform Info:
  OS: macOS (arm64-apple-darwin24.0.0)
  CPU: 10 × Apple M1 Pro
  WORD_SIZE: 64
  LLVM: libLLVM-18.1.7 (ORCJIT, apple-m1)
  GC: Built with stock GC
Threads: 1 default, 1 interactive, 1 GC (on 8 virtual cores)
Environment:
  JULIA_FORMATTER_SO = /Users/mhauru/julia_formatter.so

The whole Pkg.test(; coverage=false) log, including all dependency versions:

Details

     Testing BugMWE
      Status `/private/var/folders/wk/zmsrlr9s2cgdpdnqj5d522sw0000gr/T/jl_pc9LC0/Project.toml`
  [7a57a42e] AbstractPPL v0.13.5
  [9bb96625] BugMWE v0.1.0 `/private/tmp/mwe/BugMWE`
  [8dfed614] Test v1.11.0
      Status `/private/var/folders/wk/zmsrlr9s2cgdpdnqj5d522sw0000gr/T/jl_pc9LC0/Manifest.toml`
  [80f14c24] AbstractMCMC v5.9.0
  [7a57a42e] AbstractPPL v0.13.5
  [1520ce14] AbstractTrees v0.4.5
  [7d9f7c33] Accessors v0.1.42
  [66dad0bd] AliasTables v1.1.3
  [dce04be8] ArgCheck v2.5.0
  [198e06fe] BangBang v0.4.6
  [9718e550] Baselet v0.1.1
  [9bb96625] BugMWE v0.1.0 `/private/tmp/mwe/BugMWE`
  [a33af91c] CompositionsBase v0.1.2
  [88cd18e8] ConsoleProgressMonitor v0.1.2
  [187b0558] ConstructionBase v1.6.0
  [9a962f9c] DataAPI v1.16.0
  [864edb3b] DataStructures v0.19.1
  [e2d170a0] DataValueInterfaces v1.0.0
  [244e2a9f] DefineSingletons v0.1.2
  [b429d917] DensityInterface v0.4.0
  [ffbed154] DocStringExtensions v0.9.5
  [1a297f60] FillArrays v1.14.0
  [22cec73e] InitialValues v0.3.1
  [3587e190] InverseFunctions v0.1.17
  [92d709cd] IrrationalConstants v0.2.6
  [82899510] IteratorInterfaceExtensions v1.0.0
  [682c06a0] JSON v1.2.0
  [1d6d02ad] LeftChildRightSiblingTrees v0.2.1
  [6fdf6af0] LogDensityProblems v2.2.0
  [2ab3a3ac] LogExpFunctions v0.3.29
  [e6f89c97] LoggingExtras v1.2.0
  [1914dd2f] MacroTools v0.5.16
  [128add7d] MicroCollections v0.2.0
  [e1d29d7a] Missings v1.2.0
  [bac558e1] OrderedCollections v1.8.1
  [69de0a69] Parsers v2.8.3
  [aea7be01] PrecompileTools v1.3.3
  [21216c6a] Preferences v1.5.0
  [33c8b6b6] ProgressLogging v0.1.5
  [92933f4c] ProgressMeter v1.11.0
  [43287f4e] PtrArrays v1.3.0
  [efcf1570] Setfield v1.1.2
  [a2af1166] SortingAlgorithms v1.2.2
  [171d559e] SplittablesBase v0.1.15
  [1e83bf80] StaticArraysCore v1.4.4
  [10745b16] Statistics v1.11.1
  [82ae8749] StatsAPI v1.7.1
  [2913bbd2] StatsBase v0.34.7
  [ec057cc2] StructUtils v2.5.1
  [3783bdb8] TableTraits v1.0.1
  [bd369af6] Tables v1.12.1
  [5d786b92] TerminalLoggers v0.1.7
  [28d57a85] Transducers v0.4.85
  [56f22d72] Artifacts v1.11.0
  [2a0f44e3] Base64 v1.11.0
  [ade2ca70] Dates v1.11.0
  [8ba89e20] Distributed v1.11.0
  [9fa8497b] Future v1.11.0
  [b77e0a4c] InteractiveUtils v1.11.0
  [ac6e5ff7] JuliaSyntaxHighlighting v1.12.0
  [8f399da3] Libdl v1.11.0
  [37e2e46d] LinearAlgebra v1.12.0
  [56ddb016] Logging v1.11.0
  [d6f4376e] Markdown v1.11.0
  [de0858da] Printf v1.11.0
  [9a3f8284] Random v1.11.0
  [ea8e919c] SHA v0.7.0
  [9e88b42a] Serialization v1.11.0
  [6462fe0b] Sockets v1.11.0
  [2f01184e] SparseArrays v1.12.0
  [f489334b] StyledStrings v1.11.0
  [fa267f1f] TOML v1.0.3
  [8dfed614] Test v1.11.0
  [cf7118a7] UUIDs v1.11.0
  [4ec0a83e] Unicode v1.11.0
  [e66e0078] CompilerSupportLibraries_jll v1.3.0+1
  [4536629a] OpenBLAS_jll v0.3.29+0
  [bea87d4a] SuiteSparse_jll v7.8.3+2
  [8e850b90] libblastrampoline_jll v5.15.0+0
     Testing Running tests...
test set: Error During Test at /private/tmp/mwe/BugMWE/test/runtests.jl:21
  Got exception outside of a @test
  return type @NamedTuple{x::Vector{VarName{:x}}, y::Vector{VarName{:y, Accessors.IndexLens{Tuple{Int64}}}}, z::Vector{VarName{:z, Accessors.IndexLens{Tuple{Int64}}}}} does not match inferred return type NamedTuple{(:x, :y, :z)}
  Stacktrace:
   [1] error(s::String)
     @ Base ./error.jl:44
   [2] top-level scope
     @ /private/tmp/mwe/BugMWE/test/runtests.jl:22
   [3] macro expansion
     @ ~/.julia/juliaup/julia-1.12.1+0.aarch64.apple.darwin14/share/julia/stdlib/v1.12/Test/src/Test.jl:1776 [inlined]
   [4] macro expansion
     @ /private/tmp/mwe/BugMWE/test/runtests.jl:25 [inlined]
   [5] include(mapexpr::Function, mod::Module, _path::String)
     @ Base ./Base.jl:307
   [6] top-level scope
     @ none:6
   [7] eval(m::Module, e::Any)
     @ Core ./boot.jl:489
   [8] exec_options(opts::Base.JLOptions)
     @ Base ./client.jl:283
   [9] _start()
     @ Base ./client.jl:550
Test Summary: | Error  Total  Time
test set      |     1      1  1.9s
RNG of the outermost testset: Random.Xoshiro(0x30bbf8771a582a4e, 0xc41b9f7e01cfd15d, 0x12bd78975fab46e2, 0x5222bae47d3be59a, 0x19b54e256ac34805)
ERROR: LoadError: Some tests did not pass: 0 passed, 0 failed, 1 errored, 0 broken.
in expression starting at /private/tmp/mwe/BugMWE/test/runtests.jl:21
ERROR: Package BugMWE errored during testing
Stacktrace:
  [1] pkgerror(msg::String)
    @ Pkg.Types ~/.julia/juliaup/julia-1.12.1+0.aarch64.apple.darwin14/share/julia/stdlib/v1.12/Pkg/src/Types.jl:68
  [2] test(ctx::Pkg.Types.Context, pkgs::Vector{Pkg.Types.PackageSpec}; coverage::Bool, julia_args::Cmd, test_args::Cmd, test_fn::Nothing, force_latest_compatible_version::Bool, allow_earlier_backwards_compatible_versions::Bool, allow_reresolve::Bool)
    @ Pkg.Operations ~/.julia/juliaup/julia-1.12.1+0.aarch64.apple.darwin14/share/julia/stdlib/v1.12/Pkg/src/Operations.jl:2427
  [3] test
    @ ~/.julia/juliaup/julia-1.12.1+0.aarch64.apple.darwin14/share/julia/stdlib/v1.12/Pkg/src/Operations.jl:2280 [inlined]
  [4] test(ctx::Pkg.Types.Context, pkgs::Vector{Pkg.Types.PackageSpec}; coverage::Bool, test_fn::Nothing, julia_args::Cmd, test_args::Cmd, force_latest_compatible_version::Bool, allow_earlier_backwards_compatible_versions::Bool, allow_reresolve::Bool, kwargs::@Kwargs{io::IOContext{IO}})
    @ Pkg.API ~/.julia/juliaup/julia-1.12.1+0.aarch64.apple.darwin14/share/julia/stdlib/v1.12/Pkg/src/API.jl:484
  [5] test(pkgs::Vector{Pkg.Types.PackageSpec}; io::IOContext{IO}, kwargs::@Kwargs{coverage::Bool})
    @ Pkg.API ~/.julia/juliaup/julia-1.12.1+0.aarch64.apple.darwin14/share/julia/stdlib/v1.12/Pkg/src/API.jl:164
  [6] test(; name::Nothing, uuid::Nothing, version::Nothing, url::Nothing, rev::Nothing, path::Nothing, mode::Pkg.Types.PackageMode, subdir::Nothing, kwargs::@Kwargs{coverage::Bool})
    @ Pkg.API ~/.julia/juliaup/julia-1.12.1+0.aarch64.apple.darwin14/share/julia/stdlib/v1.12/Pkg/src/API.jl:179
  [7] top-level scope
    @ none:1
  [8] eval(m::Module, e::Any)
    @ Core ./boot.jl:489
  [9] exec_options(opts::Base.JLOptions)
    @ Base ./client.jl:283
 [10] _start()
    @ Base ./client.jl:550

[mhauru]

I still don't understand what coverage has to do with any of it, but here's a dependency-free MWE that shows the type inference regression on 1.12:

module MWE

struct VarName{sym,T}
    optic::T

    function VarName{sym}(optic::T) where {sym,T}
        new{sym,T}(optic)
    end
end

struct IndexLens{I<:Tuple}
    indices::I
end

getsym(::VarName{sym}) where {sym} = sym

@generated function unique_syms(::T) where {T}
    retval = Expr(:tuple)
    syms = [first(vn.parameters) for vn in T.parameters]
    for sym in unique(syms)
        push!(retval.args, QuoteNode(sym))
    end
    return retval
end

function group_varnames_by_symbol(vns)
    syms = unique_syms(vns)
    elements = map(collect, tuple((filter(vn -> getsym(vn) == s, vns) for s in syms)...))
    return NamedTuple{syms}(elements)
end

vns_tuple = (VarName{:x}(identity), VarName{:y}(identity), VarName{:y}(IndexLens((1,))))
println(Base.infer_return_type(group_varnames_by_symbol, (typeof(vns_tuple),)))

end

On 1.12.1:

julia> include("test/runtests.jl")
NamedTuple{(:x, :y)}

On 1.11.7:

julia> include("test/runtests.jl")
@NamedTuple{x::Vector{Main.MWE.VarName{:x, typeof(identity)}}, y::Vector{Main.MWE.VarName{:y}}}

[Seelengrab]

This is probably a duplicate of, or at least related to, #49978 since the trigger is the exact same. My guess is that something in the effects for your code changed in 1.12, making inference give up.