RFC: Vendor JuliaSyntax to avoid compatibility issues

.gitattributes

@@ -0,0 +1 @@
+vendor/*/** linguist-vendored

.github/workflows/CI.yml

@@ -19,8 +19,7 @@ jobs:
       fail-fast: false
       matrix:
         version:
-          - '1.6' # earliest supported version
-          - '1.7' # earliest version all tests run on
+          - '1.10' # earliest supported version
           - '1' # current release
           - 'beta'
           - 'nightly'

.gitignore

@@ -5,3 +5,4 @@ Manifest.toml
 Manifest-v*.toml
 /docs/build/
 dev
+!vendor/Manifest.toml

Project.toml

@@ -1,12 +1,10 @@
 name = "ExplicitImports"
 uuid = "7d51a73a-1435-4ff3-83d9-f097790105c7"
 authors = ["Eric P. Hanson"]
-version = "1.12.0"
+version = "1.12.1"
 
 [deps]
 AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c"
-Compat = "34da2185-b29b-5c13-b0c7-acf172513d20"
-JuliaSyntax = "70703baa-626e-46a2-a12c-08ffd08c73b4"
 Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
@@ -17,7 +15,6 @@ AbstractTrees = "0.4.5"
 Aqua = "0.8.4"
 Compat = "4.15"
 DataFrames = "1.6"
-JuliaSyntax = "1"
 LinearAlgebra = "<0.0.1, 1"
 Logging = "<0.0.1, 1"
 Markdown = "<0.0.1, 1"
@@ -27,12 +24,13 @@ Reexport = "1.2.2"
 TOML = "<0.0.1, 1"
 Test = "<0.0.1, 1"
 UUIDs = "<0.0.1, 1"
-julia = "1.6"
+julia = "1.10"
 
 [apps.explicit-imports-jl]
 
 [extras]
 Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
+Compat = "34da2185-b29b-5c13-b0c7-acf172513d20"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
@@ -41,4 +39,4 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
 [targets]
-test = ["Aqua", "DataFrames", "LinearAlgebra", "Logging", "UUIDs", "Reexport", "Test"]
+test = ["Aqua", "Compat", "DataFrames", "LinearAlgebra", "Logging", "UUIDs", "Reexport", "Test"]

README.md

@@ -66,45 +66,139 @@ Personally, I don't think this is always a huge issue, and that it's basically f
 julia> using ExplicitImports
 
 julia> print_explicit_imports(ExplicitImports)
-WARNING: both JuliaSyntax and Base export "parse"; uses of it in module ExplicitImports must be qualified
-  Module ExplicitImports is relying on implicit imports for 7 names. These could be explicitly imported as follows:
+  Module ExplicitImports is relying on implicit imports for 5 names. These could be explicitly imported
+  as follows:
 
   using AbstractTrees: AbstractTrees, Leaves, TreeCursor, children, nodevalue
-  using JuliaSyntax: JuliaSyntax, @K_str
+
+  Additionally, module ExplicitImports has stale explicit imports for this 1 unused name:
+
+    •  parse is unused but it was imported from ExplicitImports.JuliaSyntax at
+       /Users/eph/ExplicitImports/src/ExplicitImports.jl:9:21
+
+  Additionally, module ExplicitImports explicitly imports 1 non-public name:
+
+    •  parent is not public in AbstractTrees but it was imported from AbstractTrees at
+       /Users/eph/ExplicitImports/src/ExplicitImports.jl:12:22
 
   Additionally, module ExplicitImports has 1 self-qualified access:
 
-    •  parent was accessed as ExplicitImports.parent inside ExplicitImports at /Users/eph/ExplicitImports/src/deprecated.jl:79:21
+    •  parent was accessed as ExplicitImports.parent inside ExplicitImports at
+       /Users/eph/ExplicitImports/src/deprecated.jl:79:21
 
   Additionally, module ExplicitImports accesses 1 name from non-owner modules:
 
     •  parent has owner AbstractTrees but it was accessed from ExplicitImports at
        /Users/eph/ExplicitImports/src/deprecated.jl:79:21
-````
 
-Note: the `WARNING` is more or less harmless; the way this package is written, it will happen any time there is a clash, even if that clash is not realized in your code. I cannot figure out how to suppress it.
+  Additionally, module ExplicitImports accesses 12 non-public names:
+
+    •  Code is not public in Markdown but it was accessed via Markdown at
+       /Users/eph/ExplicitImports/src/precompile.jl:8:51
+
+    •  List is not public in Markdown but it was accessed via Markdown at
+       /Users/eph/ExplicitImports/src/precompile.jl:10:51
+
+    •  Paragraph is not public in Markdown but it was accessed via Markdown at
+       /Users/eph/ExplicitImports/src/precompile.jl:9:51
+
+    •  PkgId is not public in Base but it was accessed via Base at
+       /Users/eph/ExplicitImports/src/improper_explicit_imports.jl:49:33
+
+    •  Types is not public in Pkg but it was accessed via Pkg at
+       /Users/eph/ExplicitImports/src/main.jl:54:25
+
+    •  ioproperties is not public in Base but it was accessed via Base at
+       /Users/eph/ExplicitImports/src/main.jl:45:25
+
+    •  loaded_modules is not public in Base but it was accessed via Base at
+       /Users/eph/ExplicitImports/src/improper_explicit_imports.jl:49:51
+
+    •  loaded_modules_array is not public in Base but it was accessed via Base at
+       /Users/eph/ExplicitImports/src/ExplicitImports.jl:415:21
+
+    •  maybe_root_module is not public in Base but it was accessed via Base at
+       /Users/eph/ExplicitImports/src/improper_explicit_imports.jl:54:47
+
+    •  parse is not public in Markdown but it was accessed via Markdown at
+       /Users/eph/ExplicitImports/src/interactive_usage.jl:231:19
+
+    •  set_active_project is not public in Base but it was accessed via Base at
+       /Users/eph/ExplicitImports/src/main.jl:66:22
+
+    •  term is not public in Markdown but it was accessed via Markdown at
+       /Users/eph/ExplicitImports/src/precompile.jl:10:25
+````
 
 You can also pass `show_locations=true` for more details:
 
 ````julia
-  Module ExplicitImports is relying on implicit imports for 7 names. These could be explicitly imported as follows:
+ Module ExplicitImports is relying on implicit imports for 5 names. These could be explicitly imported
+  as follows:
 
-  using AbstractTrees: AbstractTrees # used at /Users/eph/ExplicitImports/src/parse_utilities.jl:51:10
-  using AbstractTrees: Leaves # used at /Users/eph/ExplicitImports/src/get_names_used.jl:453:17
-  using AbstractTrees: TreeCursor # used at /Users/eph/ExplicitImports/src/parse_utilities.jl:129:18
-  using AbstractTrees: children # used at /Users/eph/ExplicitImports/src/get_names_used.jl:380:26
-  using AbstractTrees: nodevalue # used at /Users/eph/ExplicitImports/src/get_names_used.jl:359:16
-  using JuliaSyntax: JuliaSyntax # used at /Users/eph/ExplicitImports/src/get_names_used.jl:439:53
-  using JuliaSyntax: @K_str # used at /Users/eph/ExplicitImports/src/get_names_used.jl:299:33
+  using AbstractTrees: AbstractTrees # used at /Users/eph/ExplicitImports/src/get_names_used.jl:79:12
+  using AbstractTrees: Leaves # used at /Users/eph/ExplicitImports/src/get_names_used.jl:462:17
+  using AbstractTrees: TreeCursor # used at /Users/eph/ExplicitImports/src/parse_utilities.jl:116:22
+  using AbstractTrees: children # used at /Users/eph/ExplicitImports/src/get_names_used.jl:225:19
+  using AbstractTrees: nodevalue # used at /Users/eph/ExplicitImports/src/parse_utilities.jl:122:34
+
+  Additionally, module ExplicitImports has stale explicit imports for this 1 unused name:
+
+    •  parse is unused but it was imported from ExplicitImports.JuliaSyntax at
+       /Users/eph/ExplicitImports/src/ExplicitImports.jl:9:21
+
+  Additionally, module ExplicitImports explicitly imports 1 non-public name:
+
+    •  parent is not public in AbstractTrees but it was imported from AbstractTrees at
+       /Users/eph/ExplicitImports/src/ExplicitImports.jl:12:22
 
   Additionally, module ExplicitImports has 1 self-qualified access:
 
-    •  parent was accessed as ExplicitImports.parent inside ExplicitImports at /Users/eph/ExplicitImports/src/deprecated.jl:79:21
+    •  parent was accessed as ExplicitImports.parent inside ExplicitImports at
+       /Users/eph/ExplicitImports/src/deprecated.jl:79:21
 
   Additionally, module ExplicitImports accesses 1 name from non-owner modules:
 
     •  parent has owner AbstractTrees but it was accessed from ExplicitImports at
        /Users/eph/ExplicitImports/src/deprecated.jl:79:21
+
+  Additionally, module ExplicitImports accesses 12 non-public names:
+
+    •  Code is not public in Markdown but it was accessed via Markdown at
+       /Users/eph/ExplicitImports/src/precompile.jl:8:51
+
+    •  List is not public in Markdown but it was accessed via Markdown at
+       /Users/eph/ExplicitImports/src/precompile.jl:10:51
+
+    •  Paragraph is not public in Markdown but it was accessed via Markdown at
+       /Users/eph/ExplicitImports/src/precompile.jl:9:51
+
+    •  PkgId is not public in Base but it was accessed via Base at
+       /Users/eph/ExplicitImports/src/improper_explicit_imports.jl:49:33
+
+    •  Types is not public in Pkg but it was accessed via Pkg at
+       /Users/eph/ExplicitImports/src/main.jl:54:25
+
+    •  ioproperties is not public in Base but it was accessed via Base at
+       /Users/eph/ExplicitImports/src/main.jl:45:25
+
+    •  loaded_modules is not public in Base but it was accessed via Base at
+       /Users/eph/ExplicitImports/src/improper_explicit_imports.jl:49:51
+
+    •  loaded_modules_array is not public in Base but it was accessed via Base at
+       /Users/eph/ExplicitImports/src/ExplicitImports.jl:415:21
+
+    •  maybe_root_module is not public in Base but it was accessed via Base at
+       /Users/eph/ExplicitImports/src/improper_explicit_imports.jl:54:47
+
+    •  parse is not public in Markdown but it was accessed via Markdown at
+       /Users/eph/ExplicitImports/src/interactive_usage.jl:231:19
+
+    •  set_active_project is not public in Base but it was accessed via Base at
+       /Users/eph/ExplicitImports/src/main.jl:66:22
+
+    •  term is not public in Markdown but it was accessed via Markdown at
+       /Users/eph/ExplicitImports/src/precompile.jl:10:25
 ````
 
 Note the paths of course will differ depending on the location of the code on your system.

docs/make.jl

@@ -6,6 +6,7 @@ DocMeta.setdocmeta!(ExplicitImports, :DocTestSetup, :(using ExplicitImports);
 
 makedocs(;
          modules=[ExplicitImports],
+         checkdocs_ignored_modules=[ExplicitImports.JuliaSyntax],
          authors="Eric P. Hanson",
          repo=Remotes.GitHub("JuliaTesting", "ExplicitImports.jl"),
          sitename="ExplicitImports.jl",

docs/src/api.md

@@ -68,3 +68,25 @@ explicit_imports_nonrecursive
 improper_qualified_accesses_nonrecursive
 improper_explicit_imports_nonrecursive
 ```
+
+## Advanced: telling ExplicitImports to ignore parts of source code
+
+You can use the `#! explicit-imports: off` and `#! explicit-imports: on` comments to disable/enable ExplicitImports from looking at parts of the code. For example, if you don't want ExplicitImports to recurse into some file, you could turn it off with:
+
+```julia
+using Foo # will be checked for implicit imports
+
+#! explicit-imports: off
+include("file.jl") # won't be checked
+#! explicit-imports: on
+
+using Bar # again will be checked
+```
+
+Note that this can of course compromise the analysis. If packages are loaded but ExplicitImports can't see them (as it has been turned off) then it will not know where the names came from. User beware!
+
+This can be paired with telling ExplicitImports to ignore submodules via [`ExplicitImports.ignore_submodules`](@ref):
+
+```@docs
+ExplicitImports.ignore_submodules
+```

docs/src/internals.md

@@ -22,4 +22,5 @@ ExplicitImports.get_names_used
 ExplicitImports.analyze_all_names
 ExplicitImports.inspect_session
 ExplicitImports.FileAnalysis
+ExplicitImports.get_default_skip_pairs
 ```

src/ExplicitImports.jl

@@ -1,17 +1,28 @@
 module ExplicitImports
 
-using JuliaSyntax, AbstractTrees
+#! explicit-imports: off
+# We vendor JuliaSyntax to avoid compatibility problems. We tell ExplicitImports to ignore
+# this inclusion since otherwise it will complain about dynamic includes and we don't really
+# want it to recurse into JS anyway.
+include(joinpath(pkgdir(ExplicitImports), "vendor", "JuliaSyntax", "src", "JuliaSyntax.jl"))
+#! explicit-imports: on
+
+using .JuliaSyntax
 # suppress warning about Base.parse collision, even though parse is never used
 # this avoids a warning when loading the package while creating an unused explicit import
 # the former occurs for all users, the latter only for developers of this package
-using JuliaSyntax: parse
+using .JuliaSyntax: parse
+
+using AbstractTrees
 using AbstractTrees: parent
 using TOML: TOML, parsefile
-using Compat: Compat, @compat
 using Markdown: Markdown
 using PrecompileTools: @setup_workload, @compile_workload
 using Pkg: Pkg
 
+# we'll borrow their `@_public` macro; if this goes away, we can get our own
+JuliaSyntax.@_public ignore_submodules
+
 export print_explicit_imports, explicit_imports, check_no_implicit_imports,
        explicit_imports_nonrecursive
 export print_explicit_imports_script
@@ -147,7 +158,7 @@ function using_statements(io::IO, rows; linewidth=80, show_locations=false,
     indent = 0
     first = true
     for (mod, row) in zip(chosen, rows)
-        @compat (; name, location) = row
+        (; name, location) = row
         if show_locations || mod !== prev_mod || separate_lines
             cur_line_width = 0
             loc = show_locations ? " # used at $(location)" : ""
@@ -304,6 +315,25 @@ function _parentmodule(mod)
     return parentmodule(mod)
 end
 
+struct ModuleDispatcher{T} end
+
+"""
+    ignore_submodules(::ModuleDispatcher{mod}) where {mod}
+
+Tell ExplicitImports to ignore direct submodules of `mod`. For example, ExplicitImports
+vendors a copy of JuliaSyntax to avoid compatibility issues. In order for ExplicitImports to ignore
+that submodule when analyzing itself, we add a method:
+
+```julia
+ExplicitImports.ignore_submodules(::ExplicitImports.ModuleDispatcher{ExplicitImports}) = (ExplicitImports.JuliaSyntax,)
+```
+
+Other packages can add methods to  `ignore_submodules` in the same way (presumably via a package extension).
+"""
+ignore_submodules(::ModuleDispatcher{T}) where {T} = ()
+
+ignore_submodules(::ModuleDispatcher{ExplicitImports}) = (JuliaSyntax,)
+
 # recurse through to find all submodules of `mod`
 function _find_submodules(mod)
     sub_modules = Set{Module}([mod])
@@ -321,7 +351,7 @@ function _find_submodules(mod)
         end
         if is_submodule
             submod = getglobal(mod, name)
-            if submod ∉ sub_modules
+            if submod ∉ sub_modules && submod ∉ ignore_submodules(ModuleDispatcher{mod}())
                 union!(sub_modules, _find_submodules(submod))
             end
         end
@@ -403,5 +433,4 @@ include("precompile.jl")
     end
 end
 
-
 end

src/checks.jl

@@ -305,9 +305,7 @@ See also: [`improper_qualified_accesses`](@ref) for programmatic access and the
 """
 function check_all_qualified_accesses_via_owners(mod::Module, file=pathof(mod);
                                                  ignore::Tuple=(),
-                                                 skip::TUPLE_MODULE_PAIRS=(Base => Core,
-                                                                           Compat => Base,
-                                                                           Compat => Core),
+                                                 skip::TUPLE_MODULE_PAIRS=get_default_skip_pairs(),
                                                  require_submodule_access=false,
                                                  allow_internal_accesses=true)
     check_file(file)
@@ -522,9 +520,7 @@ See also: [`improper_explicit_imports`](@ref) for programmatic access to such im
 """
 function check_all_explicit_imports_via_owners(mod::Module, file=pathof(mod);
                                                ignore::Tuple=(),
-                                               skip::TUPLE_MODULE_PAIRS=(Base => Core,
-                                                                         Compat => Base,
-                                                                         Compat => Core),
+                                               skip::TUPLE_MODULE_PAIRS=get_default_skip_pairs(),
                                                allow_internal_imports=true,
                                                require_submodule_import=false)
     check_file(file)

src/deprecated.jl

@@ -17,7 +17,7 @@ function stale_explicit_imports_nonrecursive(mod::Module, file=pathof(mod);
     check_file(file)
     @warn "[stale_explicit_imports_nonrecursive] deprecated in favor of `improper_explicit_imports_nonrecursive`" _id = :explicit_imports_stale_explicit_imports maxlog = 1
 
-    @compat (; unnecessary_explicit_import, tainted) = filter_to_module(file_analysis, mod)
+    (; unnecessary_explicit_import, tainted) = filter_to_module(file_analysis, mod)
     tainted && strict && return nothing
     ret = [(; nt.name, nt.location) for nt in unnecessary_explicit_import]
     return unique!(nt -> nt.name, sort!(ret))

src/get_names_used.jl

@@ -557,7 +557,7 @@ function analyze_per_usage_info(per_usage_info)
     #   2. Otherwise, if first usage is assignment, then it is local, otherwise it is global
     seen = Dict{@NamedTuple{name::Symbol,scope_path::SyntaxNodeList},Bool}()
     return map(per_usage_info) do nt
-        @compat if (; nt.name, scope_path=SyntaxNodeList(nt.scope_path)) in keys(seen)
+        if (; nt.name, scope_path=SyntaxNodeList(nt.scope_path)) in keys(seen)
             return PerUsageInfo(; nt..., first_usage_in_scope=false,
                                 external_global_name=missing,
                                 analysis_code=IgnoredNonFirst)