Merge pull request #2 from JuliaParallel/passing-tests

Make the package actually usable

Author: jpsamaroo

.github/workflows/ci.yml

@@ -14,6 +14,7 @@ concurrency:
   # Cancel intermediate builds: only pull request builds
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.ref != 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release-') || github.run_number }}
   cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
+
 jobs:
   test:
     name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}
@@ -23,6 +24,7 @@ jobs:
       matrix:
         version:
           - 'nightly'
+          - '1'
         os:
           - ubuntu-latest
           - macOS-latest
@@ -58,18 +60,17 @@ jobs:
         with:
           file: lcov.info
           token: ${{ secrets.CODECOV_TOKEN }}
+
   docs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: julia-actions/setup-julia@latest
         with:
-          # version: '1.6'
-          version: 'nightly'
-      - name: Generate docs
-        run: |
-          julia --color=yes -e 'write("Project.toml", replace(read("Project.toml", String), r"uuid = .*?\n" =>"uuid = \"47e2e46d-f89d-539d-b4ee-838fcccc9c8e\"\n"));'
-          julia --project --color=yes -e 'using Pkg; Pkg.activate("docs"); Pkg.instantiate(); Pkg.develop(PackageSpec(path = pwd()))'
-          julia --project=docs --color=yes docs/make.jl pdf
+          version: '1'
+      - name: Install dependencies
+        run: julia --project=docs/ -e 'using Pkg; Pkg.instantiate()'
+      - name: Build and deploy
         env:
-          DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: julia --project=docs/ docs/make.jl

.gitignore

@@ -0,0 +1 @@
+docs/src/changelog.md

Project.toml

@@ -1,15 +1,16 @@
 name = "DistributedNext"
 uuid = "fab6aee4-877b-4bac-a744-3eca44acbb6f"
-version = "1"
+version = "1.0.0"
 
 [deps]
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
 Sockets = "6462fe0b-24de-5631-8697-dd941f90decc"
 
 [extras]
+LibSSH = "00483490-30f8-4353-8aba-35b82f51f4d0"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["LinearAlgebra", "Test"]
+test = ["LinearAlgebra", "Test", "LibSSH"]

docs/Project.toml

@@ -1,2 +1,9 @@
 [deps]
+Changelog = "5217a498-cd5d-4ec6-b8c2-9b85a09b6e3e"
+DistributedNext = "fab6aee4-877b-4bac-a744-3eca44acbb6f"
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
+LiveServer = "16fef848-5104-11e9-1b77-fb7a48bbb589"
+Revise = "295af30f-e4ad-537b-8983-00126c2a3abe"
+
+[sources]
+DistributedNext = {path = ".."}

docs/make.jl

@@ -1,16 +1,42 @@
-using Distributed
-using Documenter: DocMeta, makedocs, deploydocs
+import Revise
+import Changelog
+using DistributedNext
+import Documenter
+using Documenter: Remotes, DocMeta, makedocs, deploydocs
 
-DocMeta.setdocmeta!(Distributed, :DocTestSetup, :(using Distributed); recursive=true)
+DocMeta.setdocmeta!(DistributedNext, :DocTestSetup, :(using DistributedNext); recursive=true)
 
-makedocs(;
-    modules = [Distributed],
-    sitename = "Distributed",
-    pages = Any[
-        "Distributed" => "index.md",
-    ],
-    checkdocs = :exports,
-    warnonly = [:cross_references],
+# Always trigger a revise to pick up the latest docstrings. This is useful when
+# working with servedocs(). If you are using servedocs(), run it like this:
+#
+# julia> servedocs(; include_dirs=["src"], skip_files=["docs/src/changelog.md"])
+#
+# Otherwise it'll get into an infinite loop as the changelog is constantly
+# regenerated and triggering LiveServer.
+Revise.revise()
+
+# Build the changelog. Note that _changelog.md is the source and changelog.md is
+# the destination. It's named that way for the vain reason of a nicer URL.
+Changelog.generate(
+    Changelog.Documenter(),
+    joinpath(@__DIR__, "src/_changelog.md"),
+    joinpath(@__DIR__, "src/changelog.md"),
+    repo="JuliaParallel/DistributedNext.jl"
 )
 
-deploydocs(repo = "github.com/JuliaLang/Distributed.jl.git")
+makedocs(;
+         repo = Remotes.GitHub("JuliaParallel", "DistributedNext.jl"),
+         format = Documenter.HTML(
+             prettyurls=get(ENV, "CI", "false") == "true",
+             size_threshold_warn=500_000,
+             size_threshold=600_000),
+         modules = [DistributedNext],
+         sitename = "DistributedNext",
+         pages = [
+             "DistributedNext" => "index.md",
+             "changelog.md"
+         ],
+         warnonly = [:missing_docs, :cross_references],
+         )
+
+deploydocs(repo = "github.com/JuliaParallel/DistributedNext.jl.git")

docs/src/_changelog.md

@@ -0,0 +1,14 @@
+```@meta
+CurrentModule = DistributedNext
+```
+
+# Changelog
+
+This documents notable changes in DistributedNext.jl. The format is based on
+[Keep a Changelog](https://keepachangelog.com).
+
+## Unreleased
+
+### Changed
+- Added a `project` argument to [`addprocs(::AbstractVector)`](@ref) to specify
+  the project of a remote worker ([#2]).

docs/src/index.md

@@ -1,53 +1,53 @@
 # [Distributed Computing](@id man-distributed)
 
 ```@docs
-Distributed
-Distributed.addprocs
-Distributed.nprocs
-Distributed.nworkers
-Distributed.procs()
-Distributed.procs(::Integer)
-Distributed.workers
-Distributed.rmprocs
-Distributed.interrupt
-Distributed.myid
-Distributed.pmap
-Distributed.RemoteException
-Distributed.ProcessExitedException
-Distributed.Future
-Distributed.RemoteChannel
-Distributed.fetch(::Distributed.Future)
-Distributed.fetch(::RemoteChannel)
-Distributed.remotecall(::Any, ::Integer, ::Any...)
-Distributed.remotecall_wait(::Any, ::Integer, ::Any...)
-Distributed.remotecall_fetch(::Any, ::Integer, ::Any...)
-Distributed.remote_do(::Any, ::Integer, ::Any...)
-Distributed.put!(::RemoteChannel, ::Any...)
-Distributed.put!(::Distributed.Future, ::Any)
-Distributed.take!(::RemoteChannel, ::Any...)
-Distributed.isready(::RemoteChannel, ::Any...)
-Distributed.isready(::Distributed.Future)
-Distributed.AbstractWorkerPool
-Distributed.WorkerPool
-Distributed.CachingPool
-Distributed.default_worker_pool
-Distributed.clear!
-Distributed.remote
-Distributed.remotecall(::Any, ::AbstractWorkerPool, ::Any...)
-Distributed.remotecall_wait(::Any, ::AbstractWorkerPool, ::Any...)
-Distributed.remotecall_fetch(::Any, ::AbstractWorkerPool, ::Any...)
-Distributed.remote_do(::Any, ::AbstractWorkerPool, ::Any...)
-Distributed.@spawn
-Distributed.@spawnat
-Distributed.@fetch
-Distributed.@fetchfrom
-Distributed.@distributed
-Distributed.@everywhere
-Distributed.remoteref_id
-Distributed.channel_from_id
-Distributed.worker_id_from_socket
-Distributed.cluster_cookie()
-Distributed.cluster_cookie(::Any)
+DistributedNext
+DistributedNext.addprocs
+DistributedNext.nprocs
+DistributedNext.nworkers
+DistributedNext.procs()
+DistributedNext.procs(::Integer)
+DistributedNext.workers
+DistributedNext.rmprocs
+DistributedNext.interrupt
+DistributedNext.myid
+DistributedNext.pmap
+DistributedNext.RemoteException
+DistributedNext.ProcessExitedException
+DistributedNext.Future
+DistributedNext.RemoteChannel
+DistributedNext.fetch(::DistributedNext.Future)
+DistributedNext.fetch(::RemoteChannel)
+DistributedNext.remotecall(::Any, ::Integer, ::Any...)
+DistributedNext.remotecall_wait(::Any, ::Integer, ::Any...)
+DistributedNext.remotecall_fetch(::Any, ::Integer, ::Any...)
+DistributedNext.remote_do(::Any, ::Integer, ::Any...)
+DistributedNext.put!(::RemoteChannel, ::Any...)
+DistributedNext.put!(::DistributedNext.Future, ::Any)
+DistributedNext.take!(::RemoteChannel, ::Any...)
+DistributedNext.isready(::RemoteChannel, ::Any...)
+DistributedNext.isready(::DistributedNext.Future)
+DistributedNext.AbstractWorkerPool
+DistributedNext.WorkerPool
+DistributedNext.CachingPool
+DistributedNext.default_worker_pool
+DistributedNext.clear!
+DistributedNext.remote
+DistributedNext.remotecall(::Any, ::AbstractWorkerPool, ::Any...)
+DistributedNext.remotecall_wait(::Any, ::AbstractWorkerPool, ::Any...)
+DistributedNext.remotecall_fetch(::Any, ::AbstractWorkerPool, ::Any...)
+DistributedNext.remote_do(::Any, ::AbstractWorkerPool, ::Any...)
+DistributedNext.@spawn
+DistributedNext.@spawnat
+DistributedNext.@fetch
+DistributedNext.@fetchfrom
+DistributedNext.@distributed
+DistributedNext.@everywhere
+DistributedNext.remoteref_id
+DistributedNext.channel_from_id
+DistributedNext.worker_id_from_socket
+DistributedNext.cluster_cookie()
+DistributedNext.cluster_cookie(::Any)
 ```
 
 ## Cluster Manager Interface
@@ -58,14 +58,14 @@ same host, and `SSHManager`, for launching on remote hosts via `ssh`. TCP/IP soc
 and transport messages between processes. It is possible for Cluster Managers to provide a different transport.
 
 ```@docs
-Distributed.ClusterManager
-Distributed.WorkerConfig
-Distributed.launch
-Distributed.manage
-Distributed.kill(::ClusterManager, ::Int, ::WorkerConfig)
-Distributed.connect(::ClusterManager, ::Int, ::WorkerConfig)
-Distributed.init_worker
-Distributed.start_worker
-Distributed.process_messages
-Distributed.default_addprocs_params
+DistributedNext.ClusterManager
+DistributedNext.WorkerConfig
+DistributedNext.launch
+DistributedNext.manage
+DistributedNext.kill(::ClusterManager, ::Int, ::WorkerConfig)
+DistributedNext.connect(::ClusterManager, ::Int, ::WorkerConfig)
+DistributedNext.init_worker
+DistributedNext.start_worker
+DistributedNext.process_messages
+DistributedNext.default_addprocs_params
 ```

src/Distributed.jl renamed to src/DistributedNext.jl

@@ -1,9 +1,10 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
 """
-Tools for distributed parallel processing.
+Tools for distributed parallel processing. This is a soft fork of Distributed.jl
+for the purposes of testing new things before merging upstream. Here be dragons!
 """
-module Distributed
+module DistributedNext
 
 # imports for extension
 import Base: getindex, wait, put!, take!, fetch, isready, push!, length,

src/macros.jl

@@ -191,7 +191,7 @@ Similar to calling `remotecall_eval(Main, procs, expr)`, but with two extra feat
 """
 macro everywhere(ex)
     procs = GlobalRef(@__MODULE__, :procs)
-    return esc(:($(Distributed).@everywhere $procs() $ex))
+    return esc(:($(DistributedNext).@everywhere $procs() $ex))
 end
 
 macro everywhere(procs, ex)

src/managers.jl

@@ -78,6 +78,10 @@ addprocs([
 
 **Keyword arguments**:
 
+* `project`: the Julia project to activate on the remote node. This *must* have
+  `DistributedNext` installed to work. Defaults to the currently active project
+  on the local node.
+
 * `tunnel`: if `true` then SSH tunneling will be used to connect to the worker from the
   master process. Default is `false`.
 
@@ -171,7 +175,8 @@ default_addprocs_params(::SSHManager) =
               :env            => [],
               :tunnel         => false,
               :multiplex      => false,
-              :max_parallel   => 10))
+              :max_parallel   => 10,
+              :project        => Base.current_project()))
 
 function launch(manager::SSHManager, params::Dict, launched::Array, launch_ntfy::Condition)
     # Launch one worker on each unique host in parallel. Additional workers are launched later.
@@ -229,6 +234,15 @@ function parse_machine(machine::AbstractString)
     (hoststr, portnum)
 end
 
+function get_worker_arg(cookie=nothing)
+    if isnothing(cookie)
+        return `-E 'using DistributedNext; DistributedNext.start_worker()'`
+    else
+        code_str = "using DistributedNext; DistributedNext.start_worker(\"$(cookie)\")"
+        return `-E $(code_str)`
+    end
+end
+
 function launch_on_machine(manager::SSHManager, machine::AbstractString, cnt, params::Dict, launched::Array, launch_ntfy::Condition)
     shell = params[:shell]
     ssh = params[:ssh]
@@ -238,8 +252,11 @@ function launch_on_machine(manager::SSHManager, machine::AbstractString, cnt, pa
     tunnel = params[:tunnel]
     multiplex = params[:multiplex]
     cmdline_cookie = params[:cmdline_cookie]
+    project = params[:project]
     env = Dict{String,String}(params[:env])
 
+    exeflags = `--project=$project $exeflags`
+
     # machine could be of the format [user@]host[:port] bind_addr[:bind_port]
     # machine format string is split on whitespace
     machine_bind = split(machine)
@@ -249,10 +266,11 @@ function launch_on_machine(manager::SSHManager, machine::AbstractString, cnt, pa
     if length(machine_bind) > 1
         exeflags = `--bind-to $(machine_bind[2]) $exeflags`
     end
+
     if cmdline_cookie
-        exeflags = `$exeflags --worker=$(cluster_cookie())`
+        exeflags = `$exeflags $(get_worker_arg(cluster_cookie()))`
     else
-        exeflags = `$exeflags --worker`
+        exeflags = `$exeflags $(get_worker_arg())`
     end
 
     host, portnum = parse_machine(machine_bind[1])
@@ -510,7 +528,7 @@ function launch(manager::LocalManager, params::Dict, launched::Array, c::Conditi
     end
 
     for i in 1:manager.np
-        cmd = `$(julia_cmd(exename)) $exeflags --bind-to $bind_to --worker`
+        cmd = `$(julia_cmd(exename)) $exeflags --bind-to $bind_to $(get_worker_arg())`
         io = open(detach(setenv(addenv(cmd, env), dir=dir)), "r+")
         write_cookie(io)
 

src/pmap.jl

@@ -240,7 +240,7 @@ Return `head`: the first `n` elements of `c`;
 and `tail`: an iterator over the remaining elements.
 
 ```jldoctest
-julia> b, c = Distributed.head_and_tail(1:10, 3)
+julia> b, c = DistributedNext.head_and_tail(1:10, 3)
 ([1, 2, 3], Base.Iterators.Rest{UnitRange{Int64}, Int64}(1:10, 3))
 
 julia> collect(c)

src/precompile.jl

@@ -1,6 +1,6 @@
-precompile(Tuple{typeof(Distributed.remotecall),Function,Int,Module,Vararg{Any, 100}})
-precompile(Tuple{typeof(Distributed.procs)})
-precompile(Tuple{typeof(Distributed.finalize_ref), Distributed.Future})
+precompile(Tuple{typeof(DistributedNext.remotecall),Function,Int,Module,Vararg{Any, 100}})
+precompile(Tuple{typeof(DistributedNext.procs)})
+precompile(Tuple{typeof(DistributedNext.finalize_ref), DistributedNext.Future})
 # This is disabled because it doesn't give much benefit
 # and the code in Distributed is poorly typed causing many invalidations
 # TODO: Maybe reenable now that Distributed is not in sysimage.