From d3b7edccf535e70452c28607dad7f9f686b92b8c Mon Sep 17 00:00:00 2001 From: Michael Schlottke-Lakemper Date: Fri, 20 Oct 2023 11:03:42 +0200 Subject: [PATCH 1/3] Allow setting a command prefix for the sysimage build command --- src/PackageCompiler.jl | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/src/PackageCompiler.jl b/src/PackageCompiler.jl index 26220f89..a743660e 100644 --- a/src/PackageCompiler.jl +++ b/src/PackageCompiler.jl @@ -220,7 +220,8 @@ function rewrite_sysimg_jl_only_needed_stdlibs(stdlibs::Vector{String}) r"stdlibs = \[(.*?)\]"s => string("stdlibs = [", join(":" .* stdlibs, ",\n"), "]")) end -function create_fresh_base_sysimage(stdlibs::Vector{String}; cpu_target::String, sysimage_build_args::Cmd) +function create_fresh_base_sysimage(stdlibs::Vector{String}; cpu_target::String, + sysimage_build_args::Cmd, sysimage_build_prefix::Cmd) tmp = mktempdir() sysimg_source_path = Base.find_source_file("sysimg.jl") base_dir = dirname(sysimg_source_path) @@ -238,7 +239,7 @@ function create_fresh_base_sysimage(stdlibs::Vector{String}; cpu_target::String, TerminalSpinners.@spin spinner begin cd(base_dir) do # Create corecompiler.ji - cmd = `$(get_julia_cmd()) --cpu-target $cpu_target + cmd = `$sysimage_build_prefix $(get_julia_cmd()) --cpu-target $cpu_target --output-ji $tmp_corecompiler_ji $sysimage_build_args $compiler_source_path` @debug "running $cmd" @@ -251,7 +252,7 @@ function create_fresh_base_sysimage(stdlibs::Vector{String}; cpu_target::String, new_sysimage_source_path = joinpath(tmp, "sysimage_packagecompiler_$(uuid1()).jl") write(new_sysimage_source_path, new_sysimage_content) try - cmd = `$(get_julia_cmd()) --cpu-target $cpu_target + cmd = `$sysimage_build_prefix $(get_julia_cmd()) --cpu-target $cpu_target --sysimage=$tmp_corecompiler_ji $sysimage_build_args --output-ji=$tmp_sys_ji $new_sysimage_source_path` @@ -303,6 +304,7 @@ function create_sysimg_object_file(object_file::String, cpu_target::String, script::Union{Nothing, String}, sysimage_build_args::Cmd, + sysimage_build_prefix::Cmd, extra_precompiles::String, incremental::Bool) julia_code_buffer = IOBuffer() @@ -429,7 +431,7 @@ function create_sysimg_object_file(object_file::String, write(outputo_file, julia_code) # Read the input via stdin to avoid hitting the maximum command line limit - cmd = `$(get_julia_cmd()) --cpu-target=$cpu_target $sysimage_build_args + cmd = `$sysimage_build_prefix $(get_julia_cmd()) --cpu-target=$cpu_target $sysimage_build_args --sysimage=$base_sysimage --project=$project --output-o=$(object_file) $outputo_file` @debug "running $cmd" @@ -487,6 +489,9 @@ compiler (can also include extra arguments to the compiler, like `-g`). - `sysimage_build_args::Cmd`: A set of command line options that is used in the Julia process building the sysimage, for example `-O1 --check-bounds=yes`. + +- `sysimage_build_prefix::Cmd`: A set of commands and command line options that will be + prefixed to the call to Julia for building the sysimage, for example `srun -n 1`. """ function create_sysimage(packages::Union{Nothing, Symbol, Vector{String}, Vector{Symbol}}=nothing; sysimage_path::String, @@ -498,6 +503,7 @@ function create_sysimage(packages::Union{Nothing, Symbol, Vector{String}, Vector cpu_target::String=NATIVE_CPU_TARGET, script::Union{Nothing, String}=nothing, sysimage_build_args::Cmd=``, + sysimage_build_prefix::Cmd=``, include_transitive_dependencies::Bool=true, # Internal args base_sysimage::Union{Nothing, String}=nothing, @@ -545,7 +551,8 @@ function create_sysimage(packages::Union{Nothing, Symbol, Vector{String}, Vector error("cannot specify `base_sysimage` when `incremental=false`") end sysimage_stdlibs = filter_stdlibs ? gather_stdlibs_project(ctx) : stdlibs_in_sysimage() - base_sysimage = create_fresh_base_sysimage(sysimage_stdlibs; cpu_target, sysimage_build_args) + base_sysimage = create_fresh_base_sysimage(sysimage_stdlibs; cpu_target, + sysimage_build_args, sysimage_build_prefix) else base_sysimage = something(base_sysimage, unsafe_string(Base.JLOptions().image_file)) end @@ -603,6 +610,7 @@ function create_sysimage(packages::Union{Nothing, Symbol, Vector{String}, Vector cpu_target, script, sysimage_build_args, + sysimage_build_prefix, extra_precompiles, incremental) object_files = [object_file] @@ -788,6 +796,9 @@ compiler (can also include extra arguments to the compiler, like `-g`). - `sysimage_build_args::Cmd`: A set of command line options that is used in the Julia process building the sysimage, for example `-O1 --check-bounds=yes`. +- `sysimage_build_prefix::Cmd`: A set of commands and command line options that will be + prefixed to the call to Julia for building the sysimage, for example `srun -n 1`. + - `script::String`: Path to a file that gets executed in the `--output-o` process. """ function create_app(package_dir::String, @@ -802,6 +813,7 @@ function create_app(package_dir::String, cpu_target::String=default_app_cpu_target(), include_lazy_artifacts::Bool=false, sysimage_build_args::Cmd=``, + sysimage_build_prefix::Cmd=``, include_transitive_dependencies::Bool=true, include_preferences::Bool=true, script::Union{Nothing, String}=nothing) @@ -851,6 +863,7 @@ function create_app(package_dir::String, precompile_statements_file, cpu_target, sysimage_build_args, + sysimage_build_prefix, include_transitive_dependencies, extra_precompiles = join(precompiles, "\n"), script) @@ -986,6 +999,9 @@ compiler (can also include extra arguments to the compiler, like `-g`). - `sysimage_build_args::Cmd`: A set of command line options that is used in the Julia process building the sysimage, for example `-O1 --check-bounds=yes`. + +- `sysimage_build_prefix::Cmd`: A set of commands and command line options that will be + prefixed to the call to Julia for building the sysimage, for example `srun -n 1`. """ function create_library(package_or_project::String, dest_dir::String; @@ -1003,6 +1019,7 @@ function create_library(package_or_project::String, cpu_target::String=default_app_cpu_target(), include_lazy_artifacts::Bool=false, sysimage_build_args::Cmd=``, + sysimage_build_prefix::Cmd=``, include_transitive_dependencies::Bool=true, include_preferences::Bool=true, script::Union{Nothing,String}=nothing @@ -1053,7 +1070,8 @@ function create_library(package_or_project::String, create_sysimage_workaround(ctx, sysimg_path, precompile_execution_file, precompile_statements_file, incremental, filter_stdlibs, cpu_target; - sysimage_build_args, include_transitive_dependencies, julia_init_c_file, + sysimage_build_args, sysimage_build_prefix, + include_transitive_dependencies, julia_init_c_file, julia_init_h_file, version, soname, script) if version !== nothing && Sys.isunix() @@ -1113,6 +1131,7 @@ function create_sysimage_workaround( filter_stdlibs::Bool, cpu_target::String; sysimage_build_args::Cmd, + sysimage_build_prefix::Cmd, include_transitive_dependencies::Bool, julia_init_c_file::Union{Nothing,String,Vector{String}}, julia_init_h_file::Union{Nothing,String,Vector{String}}, @@ -1151,6 +1170,7 @@ function create_sysimage_workaround( version, soname, sysimage_build_args, + sysimage_build_prefix, include_transitive_dependencies) return From ea3a4ca93cd014f756f3f905755864f8a5d1b125 Mon Sep 17 00:00:00 2001 From: Lars Christmann Date: Thu, 26 Oct 2023 15:47:43 +0200 Subject: [PATCH 2/3] Revert "Allow setting a command prefix for the sysimage build command" It is not necessary to provide a prefix in the build phase but it may be necessary when the script given by `precompile_execution_file` is executed, e.g. if some package requires the script to be run in an MPI context via `mpirun`. Thus, this reverts commit d3b7edccf535e70452c28607dad7f9f686b92b8c. --- src/PackageCompiler.jl | 32 ++++++-------------------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/src/PackageCompiler.jl b/src/PackageCompiler.jl index a743660e..26220f89 100644 --- a/src/PackageCompiler.jl +++ b/src/PackageCompiler.jl @@ -220,8 +220,7 @@ function rewrite_sysimg_jl_only_needed_stdlibs(stdlibs::Vector{String}) r"stdlibs = \[(.*?)\]"s => string("stdlibs = [", join(":" .* stdlibs, ",\n"), "]")) end -function create_fresh_base_sysimage(stdlibs::Vector{String}; cpu_target::String, - sysimage_build_args::Cmd, sysimage_build_prefix::Cmd) +function create_fresh_base_sysimage(stdlibs::Vector{String}; cpu_target::String, sysimage_build_args::Cmd) tmp = mktempdir() sysimg_source_path = Base.find_source_file("sysimg.jl") base_dir = dirname(sysimg_source_path) @@ -239,7 +238,7 @@ function create_fresh_base_sysimage(stdlibs::Vector{String}; cpu_target::String, TerminalSpinners.@spin spinner begin cd(base_dir) do # Create corecompiler.ji - cmd = `$sysimage_build_prefix $(get_julia_cmd()) --cpu-target $cpu_target + cmd = `$(get_julia_cmd()) --cpu-target $cpu_target --output-ji $tmp_corecompiler_ji $sysimage_build_args $compiler_source_path` @debug "running $cmd" @@ -252,7 +251,7 @@ function create_fresh_base_sysimage(stdlibs::Vector{String}; cpu_target::String, new_sysimage_source_path = joinpath(tmp, "sysimage_packagecompiler_$(uuid1()).jl") write(new_sysimage_source_path, new_sysimage_content) try - cmd = `$sysimage_build_prefix $(get_julia_cmd()) --cpu-target $cpu_target + cmd = `$(get_julia_cmd()) --cpu-target $cpu_target --sysimage=$tmp_corecompiler_ji $sysimage_build_args --output-ji=$tmp_sys_ji $new_sysimage_source_path` @@ -304,7 +303,6 @@ function create_sysimg_object_file(object_file::String, cpu_target::String, script::Union{Nothing, String}, sysimage_build_args::Cmd, - sysimage_build_prefix::Cmd, extra_precompiles::String, incremental::Bool) julia_code_buffer = IOBuffer() @@ -431,7 +429,7 @@ function create_sysimg_object_file(object_file::String, write(outputo_file, julia_code) # Read the input via stdin to avoid hitting the maximum command line limit - cmd = `$sysimage_build_prefix $(get_julia_cmd()) --cpu-target=$cpu_target $sysimage_build_args + cmd = `$(get_julia_cmd()) --cpu-target=$cpu_target $sysimage_build_args --sysimage=$base_sysimage --project=$project --output-o=$(object_file) $outputo_file` @debug "running $cmd" @@ -489,9 +487,6 @@ compiler (can also include extra arguments to the compiler, like `-g`). - `sysimage_build_args::Cmd`: A set of command line options that is used in the Julia process building the sysimage, for example `-O1 --check-bounds=yes`. - -- `sysimage_build_prefix::Cmd`: A set of commands and command line options that will be - prefixed to the call to Julia for building the sysimage, for example `srun -n 1`. """ function create_sysimage(packages::Union{Nothing, Symbol, Vector{String}, Vector{Symbol}}=nothing; sysimage_path::String, @@ -503,7 +498,6 @@ function create_sysimage(packages::Union{Nothing, Symbol, Vector{String}, Vector cpu_target::String=NATIVE_CPU_TARGET, script::Union{Nothing, String}=nothing, sysimage_build_args::Cmd=``, - sysimage_build_prefix::Cmd=``, include_transitive_dependencies::Bool=true, # Internal args base_sysimage::Union{Nothing, String}=nothing, @@ -551,8 +545,7 @@ function create_sysimage(packages::Union{Nothing, Symbol, Vector{String}, Vector error("cannot specify `base_sysimage` when `incremental=false`") end sysimage_stdlibs = filter_stdlibs ? gather_stdlibs_project(ctx) : stdlibs_in_sysimage() - base_sysimage = create_fresh_base_sysimage(sysimage_stdlibs; cpu_target, - sysimage_build_args, sysimage_build_prefix) + base_sysimage = create_fresh_base_sysimage(sysimage_stdlibs; cpu_target, sysimage_build_args) else base_sysimage = something(base_sysimage, unsafe_string(Base.JLOptions().image_file)) end @@ -610,7 +603,6 @@ function create_sysimage(packages::Union{Nothing, Symbol, Vector{String}, Vector cpu_target, script, sysimage_build_args, - sysimage_build_prefix, extra_precompiles, incremental) object_files = [object_file] @@ -796,9 +788,6 @@ compiler (can also include extra arguments to the compiler, like `-g`). - `sysimage_build_args::Cmd`: A set of command line options that is used in the Julia process building the sysimage, for example `-O1 --check-bounds=yes`. -- `sysimage_build_prefix::Cmd`: A set of commands and command line options that will be - prefixed to the call to Julia for building the sysimage, for example `srun -n 1`. - - `script::String`: Path to a file that gets executed in the `--output-o` process. """ function create_app(package_dir::String, @@ -813,7 +802,6 @@ function create_app(package_dir::String, cpu_target::String=default_app_cpu_target(), include_lazy_artifacts::Bool=false, sysimage_build_args::Cmd=``, - sysimage_build_prefix::Cmd=``, include_transitive_dependencies::Bool=true, include_preferences::Bool=true, script::Union{Nothing, String}=nothing) @@ -863,7 +851,6 @@ function create_app(package_dir::String, precompile_statements_file, cpu_target, sysimage_build_args, - sysimage_build_prefix, include_transitive_dependencies, extra_precompiles = join(precompiles, "\n"), script) @@ -999,9 +986,6 @@ compiler (can also include extra arguments to the compiler, like `-g`). - `sysimage_build_args::Cmd`: A set of command line options that is used in the Julia process building the sysimage, for example `-O1 --check-bounds=yes`. - -- `sysimage_build_prefix::Cmd`: A set of commands and command line options that will be - prefixed to the call to Julia for building the sysimage, for example `srun -n 1`. """ function create_library(package_or_project::String, dest_dir::String; @@ -1019,7 +1003,6 @@ function create_library(package_or_project::String, cpu_target::String=default_app_cpu_target(), include_lazy_artifacts::Bool=false, sysimage_build_args::Cmd=``, - sysimage_build_prefix::Cmd=``, include_transitive_dependencies::Bool=true, include_preferences::Bool=true, script::Union{Nothing,String}=nothing @@ -1070,8 +1053,7 @@ function create_library(package_or_project::String, create_sysimage_workaround(ctx, sysimg_path, precompile_execution_file, precompile_statements_file, incremental, filter_stdlibs, cpu_target; - sysimage_build_args, sysimage_build_prefix, - include_transitive_dependencies, julia_init_c_file, + sysimage_build_args, include_transitive_dependencies, julia_init_c_file, julia_init_h_file, version, soname, script) if version !== nothing && Sys.isunix() @@ -1131,7 +1113,6 @@ function create_sysimage_workaround( filter_stdlibs::Bool, cpu_target::String; sysimage_build_args::Cmd, - sysimage_build_prefix::Cmd, include_transitive_dependencies::Bool, julia_init_c_file::Union{Nothing,String,Vector{String}}, julia_init_h_file::Union{Nothing,String,Vector{String}}, @@ -1170,7 +1151,6 @@ function create_sysimage_workaround( version, soname, sysimage_build_args, - sysimage_build_prefix, include_transitive_dependencies) return From 38c58d8bea16165941ca020bb8124915b9311489 Mon Sep 17 00:00:00 2001 From: Lars Christmann Date: Thu, 26 Oct 2023 17:29:10 +0200 Subject: [PATCH 3/3] Allow setting command prefix for precompile execution This change adds some flexibility to how precompile execution files are executed by PackageCompiler to generate precompile statements. In some cases, for example when one wants to use precompile statements generated from a script that uses a package that uses MPI and requires that the Julia process is started via `mpiexec` or similar, one currently has to execute the script manually and use Julia's `--trace-compile` option to generate precompile statements and pass them manually to PackageCompiler afterwards. The new `precompile_execution_prefix` keyword argument introduced by this commit allows one to skip manual generation of precompile statements in such cases by passing the necessary prefix to PackageCompiler. --- src/PackageCompiler.jl | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/src/PackageCompiler.jl b/src/PackageCompiler.jl index 26220f89..dcc0e584 100644 --- a/src/PackageCompiler.jl +++ b/src/PackageCompiler.jl @@ -278,11 +278,12 @@ function ensurecompiled(project, packages, sysimage) return end -function run_precompilation_script(project::String, sysimg::String, precompile_file::Union{String, Nothing}, precompile_dir::String) +function run_precompilation_script(project::String, sysimg::String, precompile_prefix::Cmd, + precompile_file::Union{String, Nothing}, precompile_dir::String) tracefile, io = mktemp(precompile_dir; cleanup=false) close(io) arg = precompile_file === nothing ? `-e ''` : `$precompile_file` - cmd = `$(get_julia_cmd()) --sysimage=$(sysimg) --compile=all --trace-compile=$tracefile $arg` + cmd = `$precompile_prefix $(get_julia_cmd()) --sysimage=$(sysimg) --compile=all --trace-compile=$tracefile $arg` # --project is not propagated well with Distributed, so use environment splitter = Sys.iswindows() ? ';' : ':' @debug "run_precompilation_script: running $cmd" JULIA_LOAD_PATH = "$project$(splitter)@stdlib" @@ -299,6 +300,7 @@ function create_sysimg_object_file(object_file::String, project::String, base_sysimage::String, precompile_execution_file::Vector{String}, + precompile_execution_prefix::Cmd, precompile_statements_file::Vector{String}, cpu_target::String, script::Union{Nothing, String}, @@ -326,7 +328,8 @@ function create_sysimg_object_file(object_file::String, @debug "running precompilation execution script..." precompile_dir = mktempdir(; prefix="jl_packagecompiler_", cleanup=false) for file in (isempty(precompile_execution_file) ? (nothing,) : precompile_execution_file) - tracefile = run_precompilation_script(project, base_sysimage, file, precompile_dir) + tracefile = run_precompilation_script(project, base_sysimage, precompile_execution_prefix, + file, precompile_dir) push!(precompile_files, tracefile) end append!(precompile_files, abspath.(precompile_statements_file)) @@ -487,11 +490,16 @@ compiler (can also include extra arguments to the compiler, like `-g`). - `sysimage_build_args::Cmd`: A set of command line options that is used in the Julia process building the sysimage, for example `-O1 --check-bounds=yes`. + +- `precompile_execution_prefix::Cmd`: A set of commands and command line options that will be + prefixed to the call to Julia when executing files specified by `precompile_execution_file`, + for example `srun`. """ function create_sysimage(packages::Union{Nothing, Symbol, Vector{String}, Vector{Symbol}}=nothing; sysimage_path::String, project::String=dirname(active_project()), precompile_execution_file::Union{String, Vector{String}}=String[], + precompile_execution_prefix::Cmd=``, precompile_statements_file::Union{String, Vector{String}}=String[], incremental::Bool=true, filter_stdlibs::Bool=false, @@ -599,6 +607,7 @@ function create_sysimage(packages::Union{Nothing, Symbol, Vector{String}, Vector project, base_sysimage, precompile_execution_file, + precompile_execution_prefix, precompile_statements_file, cpu_target, script, @@ -789,11 +798,16 @@ compiler (can also include extra arguments to the compiler, like `-g`). for example `-O1 --check-bounds=yes`. - `script::String`: Path to a file that gets executed in the `--output-o` process. + +- `precompile_execution_prefix::Cmd`: A set of commands and command line options that will be + prefixed to the call to Julia when executing files specified by `precompile_execution_file`, + for example `srun`. """ function create_app(package_dir::String, app_dir::String; executables::Union{Nothing, Vector{Pair{String, String}}}=nothing, precompile_execution_file::Union{String, Vector{String}}=String[], + precompile_execution_prefix::Cmd=``, precompile_statements_file::Union{String, Vector{String}}=String[], incremental::Bool=false, filter_stdlibs::Bool=false, @@ -848,6 +862,7 @@ function create_app(package_dir::String, incremental, filter_stdlibs, precompile_execution_file, + precompile_execution_prefix, precompile_statements_file, cpu_target, sysimage_build_args, @@ -986,11 +1001,16 @@ compiler (can also include extra arguments to the compiler, like `-g`). - `sysimage_build_args::Cmd`: A set of command line options that is used in the Julia process building the sysimage, for example `-O1 --check-bounds=yes`. + +- `precompile_execution_prefix::Cmd`: A set of commands and command line options that will be + prefixed to the call to Julia when executing files specified by `precompile_execution_file`, + for example `srun`. """ function create_library(package_or_project::String, dest_dir::String; lib_name=nothing, precompile_execution_file::Union{String, Vector{String}}=String[], + precompile_execution_prefix::Cmd=``, precompile_statements_file::Union{String, Vector{String}}=String[], incremental::Bool=false, filter_stdlibs::Bool=false, @@ -1051,7 +1071,7 @@ function create_library(package_or_project::String, compat_file = get_library_filename(lib_name; version, compat_level) soname = (Sys.isunix() && !Sys.isapple()) ? compat_file : nothing - create_sysimage_workaround(ctx, sysimg_path, precompile_execution_file, + create_sysimage_workaround(ctx, sysimg_path, precompile_execution_file, precompile_execution_prefix, precompile_statements_file, incremental, filter_stdlibs, cpu_target; sysimage_build_args, include_transitive_dependencies, julia_init_c_file, julia_init_h_file, version, soname, script) @@ -1108,6 +1128,7 @@ function create_sysimage_workaround( ctx, sysimage_path::String, precompile_execution_file::Union{String, Vector{String}}, + precompile_execution_prefix::Cmd, precompile_statements_file::Union{String, Vector{String}}, incremental::Bool, filter_stdlibs::Bool, @@ -1143,6 +1164,7 @@ function create_sysimage_workaround( incremental=true, script=script, precompile_execution_file, + precompile_execution_prefix, precompile_statements_file, cpu_target, base_sysimage,