bpart: Fully switch to partitioned semantics (#57253)

This is the final PR in the binding partitions series (modulo bugs and
tweaks), i.e. it closes #54654 and thus closes #40399, which was the
original design sketch.

This thus activates the full designed semantics for binding partitions,
in particular allowing safe replacement of const bindings. It in
particular allows struct redefinitions. This thus closes
timholy/Revise.jl#18 and also closes #38584.

The biggest semantic change here is probably that this gets rid of the
notion of "resolvedness" of a binding. Previously, a lot of the behavior
of our implementation depended on when bindings were "resolved", which
could happen at basically an arbitrary point (in the compiler, in REPL
completion, in a different thread), making a lot of the semantics around
bindings ill- or at least implementation-defined. There are several
related issues in the bugtracker, so this closes #14055 closes #44604
closes #46354 closes #30277

It is also the last step to close #24569.
It also supports bindings for undef->defined transitions and thus closes
#53958 closes #54733 - however, this is not activated yet for
performance reasons and may need some further optimization.

Since resolvedness no longer exists, we need to replace it with some
hopefully more well-defined semantics. I will describe the semantics
below, but before I do I will make two notes:

1. There are a number of cases where these semantics will behave
slightly differently than the old semantics absent some other task going
around resolving random bindings.
2. The new behavior (except for the replacement stuff) was generally
permissible under the old semantics if the bindings happened to be
resolved at the right time.

With all that said, there are essentially three "strengths" of bindings:

1. Implicit Bindings: Anything implicitly obtained from `using Mod`, "no
binding", plus slightly more exotic corner cases around conflicts

2. Weakly declared bindings: Declared using `global sym` and nothing
else

3. Strongly declared bindings: Declared using `global sym::T`, `const
sym=val`, `import Mod: sym`, `using Mod: sym` or as an implicit strong
global declaration in `sym=val`, where `sym` is known to be global
(either by being at toplevle or as `global sym=val` inside a function).

In general, you always allowed to syntactically replace a weaker binding
by a stronger one (although the runtime permits arbitrary binding
deletion now, this is just a syntactic constraint to catch errors).
Second, any implicit binding can be replaced by other implicit bindings
as the result of changing the `using`'ed module. And lastly, any
constants may be replaced by any other constants (irrespective of type).

We do not currently allow replacing globals, but may consider changing
that in 1.13.

This is mostly how things used to work, as well in the absence of any
stray external binding resolutions. The most prominent difference is
probably this one:

```
set_foo!() = global foo = 1
```

In the above terminology, this now always declares a "strongly declared
binding", whereas before it declared a "weakly declared binding" that
would become strongly declared on first write to the global (unless of
course somebody had created a different strongly declared global in the
meantime). To see the difference, this is now disallowed:

```
julia> set_foo!() = global foo = 1
set_foo! (generic function with 1 method)

julia> const foo = 1
ERROR: cannot declare Main.foo constant; it was already declared global
Stacktrace:
 [1] top-level scope
   @ REPL[2]:1
```

Before it would depend on the order of binding resolution (although it
just crashes on current master for some reason - whoops, probably my
fault).

Another major change is the ambiguousness of imports. In:
```
module M1; export x; x=1; end
module M2; export x; x=2; end
using .M1, .M2
```
the binding `Main.x` is now always ambiguous and will throw on access.
Before which binding you get, would depend on resolution order. To
choose one, use an explicit import (which was the behavior you would
previously get if neither binding was resolved before both imports).

Author: Keno

Compiler/src/Compiler.jl

@@ -49,7 +49,7 @@ using Core: ABIOverride, Builtin, CodeInstance, IntrinsicFunction, MethodInstanc
 
 using Base
 using Base: @_foldable_meta, @_gc_preserve_begin, @_gc_preserve_end, @nospecializeinfer,
-    BINDING_KIND_GLOBAL, BINDING_KIND_UNDEF_CONST, BINDING_KIND_BACKDATED_CONST,
+    BINDING_KIND_GLOBAL, BINDING_KIND_UNDEF_CONST, BINDING_KIND_BACKDATED_CONST, BINDING_KIND_DECLARED,
     Base, BitVector, Bottom, Callable, DataTypeFieldDesc,
     EffectsOverride, Filter, Generator, IteratorSize, JLOptions, NUM_EFFECTS_OVERRIDES,
     OneTo, Ordering, RefValue, SizeUnknown, _NAMEDTUPLE_NAME,

Compiler/src/abstractinterpretation.jl

@@ -3464,14 +3464,7 @@ world_range(ci::CodeInfo) = WorldRange(ci.min_world, ci.max_world)
 world_range(ci::CodeInstance) = WorldRange(ci.min_world, ci.max_world)
 world_range(compact::IncrementalCompact) = world_range(compact.ir)
 
-function force_binding_resolution!(g::GlobalRef, world::UInt)
-    # Force resolution of the binding
-    # TODO: This will go away once we switch over to fully partitioned semantics
-    ccall(:jl_force_binding_resolution, Cvoid, (Any, Csize_t), g, world)
-    return nothing
-end
-
-function abstract_eval_globalref_type(g::GlobalRef, src::Union{CodeInfo, IRCode, IncrementalCompact}, retry_after_resolve::Bool=true)
+function abstract_eval_globalref_type(g::GlobalRef, src::Union{CodeInfo, IRCode, IncrementalCompact})
     worlds = world_range(src)
     partition = lookup_binding_partition(min_world(worlds), g)
     partition.max_world < max_world(worlds) && return Any
@@ -3480,25 +3473,18 @@ function abstract_eval_globalref_type(g::GlobalRef, src::Union{CodeInfo, IRCode,
         partition = lookup_binding_partition(min_world(worlds), imported_binding)
         partition.max_world < max_world(worlds) && return Any
     end
-    if is_some_guard(binding_kind(partition))
-        if retry_after_resolve
-            # This method is surprisingly hot. For performance, don't ask the runtime to resolve
-            # the binding unless necessary - doing so triggers an additional lookup, which though
-            # not super expensive is hot enough to show up in benchmarks.
-            force_binding_resolution!(g, min_world(worlds))
-            return abstract_eval_globalref_type(g, src, false)
-        end
+    kind = binding_kind(partition)
+    if is_some_guard(kind)
         # return Union{}
         return Any
     end
-    if is_some_const_binding(binding_kind(partition))
+    if is_some_const_binding(kind)
         return Const(partition_restriction(partition))
     end
-    return partition_restriction(partition)
+    return kind == BINDING_KIND_DECLARED ? Any : partition_restriction(partition)
 end
 
 function lookup_binding_partition!(interp::AbstractInterpreter, g::GlobalRef, sv::AbsIntState)
-    force_binding_resolution!(g, get_inference_world(interp))
     partition = lookup_binding_partition(get_inference_world(interp), g)
     update_valid_age!(sv, WorldRange(partition.min_world, partition.max_world))
     partition
@@ -3541,7 +3527,11 @@ function abstract_eval_partition_load(interp::AbstractInterpreter, partition::Co
         return RTEffects(rt, Union{}, Effects(EFFECTS_TOTAL, inaccessiblememonly=is_mutation_free_argtype(rt) ? ALWAYS_TRUE : ALWAYS_FALSE))
     end
 
-    rt = partition_restriction(partition)
+    if kind == BINDING_KIND_DECLARED
+        rt = Any
+    else
+        rt = partition_restriction(partition)
+    end
     return RTEffects(rt, UndefVarError, generic_getglobal_effects)
 end
 
@@ -3580,7 +3570,7 @@ function global_assignment_binding_rt_exct(interp::AbstractInterpreter, partitio
     elseif is_some_const_binding(kind)
         return Pair{Any,Any}(Bottom, ErrorException)
     end
-    ty = partition_restriction(partition)
+    ty = kind == BINDING_KIND_DECLARED ? Any : partition_restriction(partition)
     wnewty = widenconst(newty)
     if !hasintersect(wnewty, ty)
         return Pair{Any,Any}(Bottom, TypeError)

Compiler/src/ssair/ir.jl

@@ -581,7 +581,7 @@ function is_relevant_expr(e::Expr)
                       :foreigncall, :isdefined, :copyast,
                       :throw_undef_if_not,
                       :cfunction, :method, :pop_exception,
-                      :leave,
+                      :leave, :const, :globaldecl,
                       :new_opaque_closure)
 end
 

Compiler/src/ssair/verify.jl

@@ -61,7 +61,6 @@ function check_op(ir::IRCode, domtree::DomTree, @nospecialize(op), use_bb::Int,
             raise_error()
         end
     elseif isa(op, GlobalRef)
-        force_binding_resolution!(op, min_world(ir.valid_worlds))
         bpart = lookup_binding_partition(min_world(ir.valid_worlds), op)
         while is_some_imported(binding_kind(bpart)) && max_world(ir.valid_worlds) <= bpart.max_world
             imported_binding = partition_restriction(bpart)::Core.Binding

Compiler/src/validation.jl

@@ -22,7 +22,7 @@ const VALID_EXPR_HEADS = IdDict{Symbol,UnitRange{Int}}(
     :copyast => 1:1,
     :meta => 0:typemax(Int),
     :global => 1:1,
-    :globaldecl => 2:2,
+    :globaldecl => 1:2,
     :foreigncall => 5:typemax(Int), # name, RT, AT, nreq, (cconv, effects), args..., roots...
     :cfunction => 5:5,
     :isdefined => 1:2,

Compiler/test/effects.jl

@@ -378,32 +378,27 @@ let effects = Base.infer_effects(; optimize=false) do
 end
 
 # we should taint `nothrow` if the binding doesn't exist and isn't fixed yet,
-# as the cached effects can be easily wrong otherwise
-# since the inference currently doesn't track "world-age" of global variables
-@eval global_assignment_undefinedyet() = $(GlobalRef(@__MODULE__, :UNDEFINEDYET)) = 42
 setglobal!_nothrow_undefinedyet() = setglobal!(@__MODULE__, :UNDEFINEDYET, 42)
-let effects = Base.infer_effects() do
-        global_assignment_undefinedyet()
-    end
+let effects = Base.infer_effects(setglobal!_nothrow_undefinedyet)
     @test !Compiler.is_nothrow(effects)
 end
-let effects = Base.infer_effects() do
-        setglobal!_nothrow_undefinedyet()
-    end
-    @test !Compiler.is_nothrow(effects)
+@test_throws ErrorException setglobal!_nothrow_undefinedyet()
+# This declares the binding as ::Any
+@eval global_assignment_undefinedyet() = $(GlobalRef(@__MODULE__, :UNDEFINEDYET)) = 42
+let effects = Base.infer_effects(global_assignment_undefinedyet)
+    @test Compiler.is_nothrow(effects)
 end
-global UNDEFINEDYET::String = "0"
-let effects = Base.infer_effects() do
-        global_assignment_undefinedyet()
-    end
+# Again with type mismatch
+global UNDEFINEDYET2::String = "0"
+setglobal!_nothrow_undefinedyet2() = setglobal!(@__MODULE__, :UNDEFINEDYET2, 42)
+@eval global_assignment_undefinedyet2() = $(GlobalRef(@__MODULE__, :UNDEFINEDYET2)) = 42
+let effects = Base.infer_effects(global_assignment_undefinedyet2)
     @test !Compiler.is_nothrow(effects)
 end
-let effects = Base.infer_effects() do
-        setglobal!_nothrow_undefinedyet()
-    end
+let effects = Base.infer_effects(setglobal!_nothrow_undefinedyet2)
     @test !Compiler.is_nothrow(effects)
 end
-@test_throws Union{ErrorException,TypeError} setglobal!_nothrow_undefinedyet() # TODO: what kind of error should this be?
+@test_throws TypeError setglobal!_nothrow_undefinedyet2()
 
 # Nothrow for setfield!
 mutable struct SetfieldNothrow

Compiler/test/inference.jl

@@ -6160,7 +6160,7 @@ end === Int
     swapglobal!(@__MODULE__, :swapglobal!_xxx, x)
 end === Union{}
 
-global swapglobal!_must_throw
+eval(Expr(:const, :swapglobal!_must_throw))
 @newinterp SwapGlobalInterp
 Compiler.InferenceParams(::SwapGlobalInterp) = Compiler.InferenceParams(; assume_bindings_static=true)
 function func_swapglobal!_must_throw(x)

base/boot.jl

@@ -229,7 +229,7 @@ export
     Expr, QuoteNode, LineNumberNode, GlobalRef,
     # object model functions
     fieldtype, getfield, setfield!, swapfield!, modifyfield!, replacefield!, setfieldonce!,
-    nfields, throw, tuple, ===, isdefined, eval,
+    nfields, throw, tuple, ===, isdefined,
     # access to globals
     getglobal, setglobal!, swapglobal!, modifyglobal!, replaceglobal!, setglobalonce!, isdefinedglobal,
     # ifelse, sizeof    # not exported, to avoid conflicting with Base

base/client.jl

@@ -267,9 +267,8 @@ function exec_options(opts)
         let Distributed = require(PkgId(UUID((0x8ba89e20_285c_5b6f, 0x9357_94700520ee1b)), "Distributed"))
             Core.eval(MainInclude, :(const Distributed = $Distributed))
             Core.eval(Main, :(using Base.MainInclude.Distributed))
+            invokelatest(Distributed.process_opts, opts)
         end
-
-        invokelatest(Main.Distributed.process_opts, opts)
     end
 
     interactiveinput = (repl || is_interactive::Bool) && isa(stdin, TTY)

base/deprecated.jl

@@ -531,4 +531,29 @@ end
 
 # BEGIN 1.12 deprecations
 
+@deprecate isbindingresolved(m::Module, var::Symbol) true false
+
+"""
+    isbindingresolved(m::Module, s::Symbol) -> Bool
+
+Returns whether the binding of a symbol in a module is resolved.
+
+See also: [`isexported`](@ref), [`ispublic`](@ref), [`isdeprecated`](@ref)
+
+```jldoctest
+julia> module Mod
+           foo() = 17
+       end
+Mod
+
+julia> Base.isbindingresolved(Mod, :foo)
+true
+```
+
+!!! warning
+    This function is deprecated. The concept of binding "resolvedness" was removed in Julia 1.12.
+    The function now always returns `true`.
+"""
+isbindingresolved
+
 # END 1.12 deprecations

base/invalidation.jl

@@ -113,32 +113,56 @@ function invalidate_method_for_globalref!(gr::GlobalRef, method::Method, invalid
     end
 end
 
-function invalidate_code_for_globalref!(gr::GlobalRef, invalidated_bpart::Core.BindingPartition, new_max_world::UInt)
-    b = convert(Core.Binding, gr)
-    try
-        valid_in_valuepos = false
-        foreach_module_mtable(gr.mod, new_max_world) do mt::Core.MethodTable
-            for method in MethodList(mt)
-                invalidate_method_for_globalref!(gr, method, invalidated_bpart, new_max_world)
+const BINDING_FLAG_EXPORTP = 0x2
+
+function invalidate_code_for_globalref!(b::Core.Binding, invalidated_bpart::Core.BindingPartition, new_bpart::Union{Core.BindingPartition, Nothing}, new_max_world::UInt)
+    gr = b.globalref
+    if is_some_guard(binding_kind(invalidated_bpart))
+        # TODO: We may want to invalidate for these anyway, since they have performance implications
+        return
+    end
+    foreach_module_mtable(gr.mod, new_max_world) do mt::Core.MethodTable
+        for method in MethodList(mt)
+            invalidate_method_for_globalref!(gr, method, invalidated_bpart, new_max_world)
+        end
+        return true
+    end
+    if isdefined(b, :backedges)
+        for edge in b.backedges
+            if isa(edge, CodeInstance)
+                ccall(:jl_invalidate_code_instance, Cvoid, (Any, UInt), edge, new_max_world)
+            elseif isa(edge, Core.Binding)
+                isdefined(edge, :partitions) || continue
+                latest_bpart = edge.partitions
+                latest_bpart.max_world == typemax(UInt) || continue
+                is_some_imported(binding_kind(latest_bpart)) || continue
+                partition_restriction(latest_bpart) === b || continue
+                invalidate_code_for_globalref!(edge, latest_bpart, nothing, new_max_world)
+            else
+                invalidate_method_for_globalref!(gr, edge::Method, invalidated_bpart, new_max_world)
             end
-            return true
         end
-        b = convert(Core.Binding, gr)
-        if isdefined(b, :backedges)
-            for edge in b.backedges
-                if isa(edge, CodeInstance)
-                    ccall(:jl_invalidate_code_instance, Cvoid, (Any, UInt), edge, new_max_world)
-                else
-                    invalidate_method_for_globalref!(gr, edge::Method, invalidated_bpart, new_max_world)
-                end
+    end
+    if (b.flags & BINDING_FLAG_EXPORTP) != 0
+        # This binding was exported - we need to check all modules that `using` us to see if they
+        # have an implicit binding to us.
+        usings_backedges = ccall(:jl_get_module_usings_backedges, Any, (Any,), gr.mod)
+        if usings_backedges !== nothing
+            for user in usings_backedges::Vector{Any}
+                user_binding = ccall(:jl_get_module_binding_or_nothing, Any, (Any, Any), user, gr.name)
+                user_binding === nothing && continue
+                isdefined(user_binding, :partitions) || continue
+                latest_bpart = user_binding.partitions
+                latest_bpart.max_world == typemax(UInt) || continue
+                is_some_imported(binding_kind(latest_bpart)) || continue
+                partition_restriction(latest_bpart) === b || continue
+                invalidate_code_for_globalref!(convert(Core.Binding, user_binding), latest_bpart, nothing, new_max_world)
             end
         end
-    catch err
-        bt = catch_backtrace()
-        invokelatest(Base.println, "Internal Error during invalidation:")
-        invokelatest(Base.display_error, err, bt)
     end
 end
+invalidate_code_for_globalref!(gr::GlobalRef, invalidated_bpart::Core.BindingPartition, new_bpart::Core.BindingPartition, new_max_world::UInt) =
+    invalidate_code_for_globalref!(convert(Core.Binding, gr), invalidated_bpart, new_bpart, new_max_world)
 
 gr_needs_backedge_in_module(gr::GlobalRef, mod::Module) = gr.mod !== mod
 

base/reflection.jl

@@ -46,92 +46,6 @@ function code_lowered(@nospecialize(f), @nospecialize(t=Tuple); generated::Bool=
     return ret
 end
 
-# high-level, more convenient method lookup functions
-
-function visit(f, mt::Core.MethodTable)
-    mt.defs !== nothing && visit(f, mt.defs)
-    nothing
-end
-function visit(f, mc::Core.TypeMapLevel)
-    function avisit(f, e::Memory{Any})
-        for i in 2:2:length(e)
-            isassigned(e, i) || continue
-            ei = e[i]
-            if ei isa Memory{Any}
-                for j in 2:2:length(ei)
-                    isassigned(ei, j) || continue
-                    visit(f, ei[j])
-                end
-            else
-                visit(f, ei)
-            end
-        end
-    end
-    if mc.targ !== nothing
-        avisit(f, mc.targ::Memory{Any})
-    end
-    if mc.arg1 !== nothing
-        avisit(f, mc.arg1::Memory{Any})
-    end
-    if mc.tname !== nothing
-        avisit(f, mc.tname::Memory{Any})
-    end
-    if mc.name1 !== nothing
-        avisit(f, mc.name1::Memory{Any})
-    end
-    mc.list !== nothing && visit(f, mc.list)
-    mc.any !== nothing && visit(f, mc.any)
-    nothing
-end
-function visit(f, d::Core.TypeMapEntry)
-    while d !== nothing
-        f(d.func)
-        d = d.next
-    end
-    nothing
-end
-struct MethodSpecializations
-    specializations::Union{Nothing, Core.MethodInstance, Core.SimpleVector}
-end
-"""
-    specializations(m::Method) → itr
-
-Return an iterator `itr` of all compiler-generated specializations of `m`.
-"""
-specializations(m::Method) = MethodSpecializations(isdefined(m, :specializations) ? m.specializations : nothing)
-function iterate(specs::MethodSpecializations)
-    s = specs.specializations
-    s === nothing && return nothing
-    isa(s, Core.MethodInstance) && return (s, nothing)
-    return iterate(specs, 0)
-end
-iterate(specs::MethodSpecializations, ::Nothing) = nothing
-function iterate(specs::MethodSpecializations, i::Int)
-    s = specs.specializations::Core.SimpleVector
-    n = length(s)
-    i >= n && return nothing
-    item = nothing
-    while i < n && item === nothing
-        item = s[i+=1]
-    end
-    item === nothing && return nothing
-    return (item, i)
-end
-length(specs::MethodSpecializations) = count(Returns(true), specs)
-
-function length(mt::Core.MethodTable)
-    n = 0
-    visit(mt) do m
-        n += 1
-    end
-    return n::Int
-end
-isempty(mt::Core.MethodTable) = (mt.defs === nothing)
-
-uncompressed_ir(m::Method) = isdefined(m, :source) ? _uncompressed_ir(m) :
-                             isdefined(m, :generator) ? error("Method is @generated; try `code_lowered` instead.") :
-                             error("Code for this Method is not available.")
-
 # for backwards compat
 const uncompressed_ast = uncompressed_ir
 const _uncompressed_ast = _uncompressed_ir