Skip to content

Commit 6edc0a9

Browse files
JamesWrigleyJamesWrigley
committed
Use LibSSH.jl to set up an SSH server for the SSHManager tests
Also enables it by default.
1 parent e150bce commit 6edc0a9

File tree

2 files changed

+73
-73
lines changed

2 files changed

+73
-73
lines changed

‎Project.toml

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,16 @@
11
name = "DistributedNext"
22
uuid = "fab6aee4-877b-4bac-a744-3eca44acbb6f"
3-
version = "1"
3+
version = "1.0.0"
44

55
[deps]
66
Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
77
Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
88
Sockets = "6462fe0b-24de-5631-8697-dd941f90decc"
99

1010
[extras]
11+
LibSSH = "00483490-30f8-4353-8aba-35b82f51f4d0"
1112
LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
1213
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
1314

1415
[targets]
15-
test = ["LinearAlgebra", "Test"]
16+
test = ["LinearAlgebra", "Test", "LibSSH"]

‎test/distributed_exec.jl

Lines changed: 70 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@
33
using Test, DistributedNext, Random, Serialization, Sockets
44
import DistributedNext: launch, manage
55

6+
import LibSSH as ssh
7+
import LibSSH.Demo: DemoServer
8+
69
@test cluster_cookie() isa String
710

811
include(joinpath(Sys.BINDIR, "..", "share", "julia", "test", "testenv.jl"))
@@ -743,11 +746,9 @@ finally
743746
DistributedNext.default_worker_pool!(wp_default)
744747
end
745748

746-
# The below block of tests are usually run only on local development systems, since:
747-
# - tests which print errors
748-
# - addprocs tests are memory intensive
749-
# - ssh addprocs requires sshd to be running locally with passwordless login enabled.
750-
# The test block is enabled by defining env JULIA_TESTFULL=1
749+
# The below block of tests are usually run only on local development systems,
750+
# since they print errors. The test block is enabled by defining env
751+
# JULIA_TESTFULL=1.
751752

752753
DoFullTest = Base.get_bool_env("JULIA_TESTFULL", false)
753754

@@ -772,6 +773,7 @@ if DoFullTest
772773
end
773774
@test workers() == all_w
774775
@test all([p == remotecall_fetch(myid, p) for p in all_w])
776+
end
775777

776778
if Sys.isunix() # aka have ssh
777779
function test_n_remove_pids(new_pids)
@@ -791,75 +793,72 @@ if Sys.isunix() # aka have ssh
791793
remotecall_fetch(rmprocs, 1, new_pids)
792794
end
793795

794-
print("\n\nTesting SSHManager. A minimum of 4GB of RAM is recommended.\n")
795-
print("Please ensure: \n")
796-
print("1) sshd is running locally with passwordless login enabled.\n")
797-
print("2) Env variable USER is defined and is the ssh user.\n")
798-
print("3) Port 9300 is not in use.\n")
799-
800-
sshflags = `-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o LogLevel=ERROR `
801-
#Issue #9951
802-
hosts=[]
803-
localhost_aliases = ["localhost", string(getipaddr()), "127.0.0.1"]
804-
num_workers = parse(Int,(get(ENV, "JULIA_ADDPROCS_NUM", "9")))
805-
806-
for i in 1:(num_workers/length(localhost_aliases))
807-
append!(hosts, localhost_aliases)
808-
end
809-
810-
print("\nTesting SSH addprocs with $(length(hosts)) workers...\n")
811-
new_pids = addprocs_with_testenv(hosts; sshflags=sshflags)
812-
@test length(new_pids) == length(hosts)
813-
test_n_remove_pids(new_pids)
814-
815-
print("\nMixed ssh addprocs with :auto\n")
816-
new_pids = addprocs_with_testenv(["localhost", ("127.0.0.1", :auto), "localhost"]; sshflags=sshflags)
817-
@test length(new_pids) == (2 + Sys.CPU_THREADS)
818-
test_n_remove_pids(new_pids)
819-
820-
print("\nMixed ssh addprocs with numeric counts\n")
821-
new_pids = addprocs_with_testenv([("localhost", 2), ("127.0.0.1", 2), "localhost"]; sshflags=sshflags)
822-
@test length(new_pids) == 5
823-
test_n_remove_pids(new_pids)
824-
825-
print("\nssh addprocs with tunnel\n")
826-
new_pids = addprocs_with_testenv([("localhost", num_workers)]; tunnel=true, sshflags=sshflags)
827-
@test length(new_pids) == num_workers
828-
test_n_remove_pids(new_pids)
829-
830-
print("\nssh addprocs with tunnel (SSH multiplexing)\n")
831-
new_pids = addprocs_with_testenv([("localhost", num_workers)]; tunnel=true, multiplex=true, sshflags=sshflags)
832-
@test length(new_pids) == num_workers
833-
controlpath = joinpath(homedir(), ".ssh", "julia-$(ENV["USER"])@localhost:22")
834-
@test issocket(controlpath)
835-
test_n_remove_pids(new_pids)
836-
@test :ok == timedwait(()->!issocket(controlpath), 10.0; pollint=0.5)
837-
838-
print("\nAll supported formats for hostname\n")
839-
h1 = "localhost"
840-
user = ENV["USER"]
841-
h2 = "$user@$h1"
842-
h3 = "$h2:22"
843-
h4 = "$h3 $(string(getipaddr()))"
844-
h5 = "$h4:9300"
845-
846-
new_pids = addprocs_with_testenv([h1, h2, h3, h4, h5]; sshflags=sshflags)
847-
@test length(new_pids) == 5
848-
test_n_remove_pids(new_pids)
849-
850-
print("\nkeyword arg exename\n")
851-
for exename in [`$(joinpath(Sys.BINDIR, Base.julia_exename()))`, "$(joinpath(Sys.BINDIR, Base.julia_exename()))"]
852-
for addp_func in [()->addprocs_with_testenv(["localhost"]; exename=exename, exeflags=test_exeflags, sshflags=sshflags),
853-
()->addprocs_with_testenv(1; exename=exename, exeflags=test_exeflags)]
854-
855-
local new_pids = addp_func()
856-
@test length(new_pids) == 1
857-
test_n_remove_pids(new_pids)
796+
println("\n\nTesting SSHManager. A minimum of 4GB of RAM is recommended.")
797+
println("Please ensure port 9300 and 2222 are not in use.")
798+
799+
DemoServer(2222; auth_methods=[ssh.AuthMethod_None], allow_auth_none=true, verbose=false, timeout=3600) do
800+
sshflags = `-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o LogLevel=ERROR -p 2222 `
801+
#Issue #9951
802+
hosts=[]
803+
localhost_aliases = ["localhost", string(getipaddr()), "127.0.0.1"]
804+
num_workers = parse(Int,(get(ENV, "JULIA_ADDPROCS_NUM", "9")))
805+
806+
for i in 1:(num_workers/length(localhost_aliases))
807+
append!(hosts, localhost_aliases)
858808
end
859-
end
860809

810+
print("\nTesting SSH addprocs with $(length(hosts)) workers...\n")
811+
new_pids = addprocs_with_testenv(hosts; sshflags=sshflags)
812+
@test length(new_pids) == length(hosts)
813+
test_n_remove_pids(new_pids)
814+
815+
print("\nMixed ssh addprocs with :auto\n")
816+
new_pids = addprocs_with_testenv(["localhost", ("127.0.0.1", :auto), "localhost"]; sshflags=sshflags)
817+
@test length(new_pids) == (2 + Sys.CPU_THREADS)
818+
test_n_remove_pids(new_pids)
819+
820+
print("\nMixed ssh addprocs with numeric counts\n")
821+
new_pids = addprocs_with_testenv([("localhost", 2), ("127.0.0.1", 2), "localhost"]; sshflags=sshflags)
822+
@test length(new_pids) == 5
823+
test_n_remove_pids(new_pids)
824+
825+
print("\nssh addprocs with tunnel\n")
826+
new_pids = addprocs_with_testenv([("localhost", num_workers)]; tunnel=true, sshflags=sshflags)
827+
@test length(new_pids) == num_workers
828+
test_n_remove_pids(new_pids)
829+
830+
print("\nssh addprocs with tunnel (SSH multiplexing)\n")
831+
new_pids = addprocs_with_testenv([("localhost", num_workers)]; tunnel=true, multiplex=true, sshflags=sshflags)
832+
@test length(new_pids) == num_workers
833+
controlpath = joinpath(homedir(), ".ssh", "julia-$(ENV["USER"])@localhost:2222")
834+
@test issocket(controlpath)
835+
test_n_remove_pids(new_pids)
836+
@test :ok == timedwait(()->!issocket(controlpath), 10.0; pollint=0.5)
837+
838+
print("\nAll supported formats for hostname\n")
839+
h1 = "localhost"
840+
user = ENV["USER"]
841+
h2 = "$user@$h1"
842+
h3 = "$h2:2222"
843+
h4 = "$h3 $(string(getipaddr()))"
844+
h5 = "$h4:9300"
845+
846+
new_pids = addprocs_with_testenv([h1, h2, h3, h4, h5]; sshflags=sshflags)
847+
@test length(new_pids) == 5
848+
test_n_remove_pids(new_pids)
849+
850+
print("\nkeyword arg exename\n")
851+
for exename in [`$(joinpath(Sys.BINDIR, Base.julia_exename()))`, "$(joinpath(Sys.BINDIR, Base.julia_exename()))"]
852+
for addp_func in [()->addprocs_with_testenv(["localhost"]; exename=exename, exeflags=test_exeflags, sshflags=sshflags),
853+
()->addprocs_with_testenv(1; exename=exename, exeflags=test_exeflags)]
854+
855+
local new_pids = addp_func()
856+
@test length(new_pids) == 1
857+
test_n_remove_pids(new_pids)
858+
end
859+
end
860+
end
861861
end # unix-only
862-
end # full-test
863862

864863
let t = @task 42
865864
schedule(t, ErrorException(""), error=true)

0 commit comments

Comments
 (0)