From 9852b766c5b777e600658fed60f0c9e06edb534b Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Wed, 11 Mar 2020 15:37:39 -0400 Subject: [PATCH 1/3] implement NVTXT --- src/extras/extras.jl | 3 ++ src/extras/timeline.jl | 97 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 src/extras/timeline.jl diff --git a/src/extras/extras.jl b/src/extras/extras.jl index 5a2534080..0dd235285 100644 --- a/src/extras/extras.jl +++ b/src/extras/extras.jl @@ -4,4 +4,7 @@ include("loopinfo.jl") using .LoopInfo export @unroll +include("timeline.jl") +using .Timeline + end # module diff --git a/src/extras/timeline.jl b/src/extras/timeline.jl new file mode 100644 index 000000000..2e5e5de5e --- /dev/null +++ b/src/extras/timeline.jl @@ -0,0 +1,97 @@ +module Timeline + +module NVTXT + const LOG_FILE=Ref{IOStream}() + const SHOULD_LOG=Ref{Bool}(false) + + function __init__() + if haskey(ENV, "KERNELABSTRACTIONS_TIMELINE") + SHOULD_LOG[] = true + else + SHOULD_LOG[] = false + return + end + pid = Libc.getpid() + LOG_FILE[] = open("ka-$pid.nvtxt", "w") + initialize() + atexit() do + close(LOG_FILE[]) + end + end + + function initialize() + SHOULD_LOG[] || return + io = LOG_FILE[] + pid = Libc.getpid() + print(io, """ + SetFileDisplayName, KernelAbstractions + @RangeStartEnd, Start, End, ThreadId, Message + ProcessId = $pid + CategoryId = 1 + Color = Blue + TimeBase = ClockMonotonicRaw + @RangePush, Time, ThreadId, Message + ProcessId = $pid + CategoryId = 1 + Color = Blue + TimeBase = ClockMonotonicRaw + @RangePop, Time, ThreadId + ProcessId = $pid + TimeBase = ClockMonotonicRaw + @Marker, Time, ThreadId, Message + ProcessId = $pid + CategoryId = 1 + Color = Blue + TimeBase = ClockMonotonicRaw + """) + end + + function push_range(msg) + SHOULD_LOG[] || return + time = time_ns() + io = LOG_FILE[] + print(io, "RangePush, ") + print(io, time) + println(io, ", ", Base.Threads.threadid(), ", \"", msg, "\"") + end + + function pop_range() + SHOULD_LOG[] || return + time = time_ns() + io = LOG_FILE[] + print(io, "RangePop, ") + print(io, time) + println(io, ", ", Base.Threads.threadid()) + end + + struct Range + start::UInt64 + msg::String + end + + start_range(msg::String) = Range(time_ns(), msg) + function end_range(r::Range) + SHOULD_LOG[] || return + time = time_ns() + io = LOG_FILE[] + print(io, "RangeStartEnd, ") + show(io, r.start) + print(io, ", ") + show(io, time) + println(io, ", ", Base.Threads.threadid(), ", \"", r.msg, "\"") + end + + function mark(msg::String) + SHOULD_LOG[] || return + time = time_ns() + io = LOG_FILE[] + print(io, "Marker, ") + show(io, time) + println(io, ", ", Base.Threads.threadid(), ", \"", msg, "\"") + end +end # NVTXT + +function range_push end +function mark end + +end \ No newline at end of file From 22564a2e7d7e6a4cfe347399a99bb5a9d5df366d Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Wed, 11 Mar 2020 18:32:17 -0400 Subject: [PATCH 2/3] switch to manual --- src/extras/timeline.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/extras/timeline.jl b/src/extras/timeline.jl index 2e5e5de5e..8b7e0f699 100644 --- a/src/extras/timeline.jl +++ b/src/extras/timeline.jl @@ -29,20 +29,20 @@ module NVTXT ProcessId = $pid CategoryId = 1 Color = Blue - TimeBase = ClockMonotonicRaw + TimeBase = Manual @RangePush, Time, ThreadId, Message ProcessId = $pid CategoryId = 1 Color = Blue - TimeBase = ClockMonotonicRaw + TimeBase = Manual @RangePop, Time, ThreadId ProcessId = $pid - TimeBase = ClockMonotonicRaw + TimeBase = Manual @Marker, Time, ThreadId, Message ProcessId = $pid CategoryId = 1 Color = Blue - TimeBase = ClockMonotonicRaw + TimeBase = Manual """) end From 222cc303759fd65239171391bc87d0abbdbe6ce6 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Fri, 13 Mar 2020 21:47:43 -0400 Subject: [PATCH 3/3] switch to NVTX when available --- src/KernelAbstractions.jl | 16 +++++++------- src/backends/cpu.jl | 9 ++++++-- src/extras/extras.jl | 1 + src/extras/timeline.jl | 44 ++++++++++++++++++++++++++++++++++++--- 4 files changed, 58 insertions(+), 12 deletions(-) diff --git a/src/KernelAbstractions.jl b/src/KernelAbstractions.jl index eaa24dd0c..a29533b2b 100644 --- a/src/KernelAbstractions.jl +++ b/src/KernelAbstractions.jl @@ -430,6 +430,15 @@ end end end +### +# Extras +# - LoopInfo +# - Timeline +### + +include("extras/extras.jl") +import .Extras.Timeline + ### # Backends/Implementation ### @@ -442,11 +451,4 @@ include("backends/cpu.jl") @init @require CUDAnative="be33ccc6-a3ff-5ff2-a52e-74243cff1e17" begin include("backends/cuda.jl") end - -### -# Extras -# - LoopInfo -### - -include("extras/extras.jl") end #module diff --git a/src/backends/cpu.jl b/src/backends/cpu.jl index 1488d53a1..b0345384c 100644 --- a/src/backends/cpu.jl +++ b/src/backends/cpu.jl @@ -8,8 +8,11 @@ end function Event(f, args...; dependencies=nothing, progress=nothing) T = Threads.@spawn begin - wait(MultiEvent(dependencies), progress) - f(args...) + Timeline.range "Event($(nameof(f))" begin + wait(MultiEvent(dependencies), progress) + Timeline.mark("Event($(nameof(f))) waiting done") + f(args...) + end end return CPUEvent(T) end @@ -79,6 +82,7 @@ function __run(obj, ndrange, iterspace, args, ::Val{dynamic}) where dynamic Nthreads = N len, rem = 1, 0 end + Timeline.@range string(nameof(obj.f)) begin if Nthreads == 1 __thread_run(1, len, rem, obj, ndrange, iterspace, args, Val(dynamic)) else @@ -86,6 +90,7 @@ function __run(obj, ndrange, iterspace, args, ::Val{dynamic}) where dynamic Threads.@spawn __thread_run(tid, len, rem, obj, ndrange, iterspace, args, Val(dynamic)) end end + end # Timeline return nothing end diff --git a/src/extras/extras.jl b/src/extras/extras.jl index 0dd235285..a9e088fee 100644 --- a/src/extras/extras.jl +++ b/src/extras/extras.jl @@ -6,5 +6,6 @@ export @unroll include("timeline.jl") using .Timeline +export Timeline end # module diff --git a/src/extras/timeline.jl b/src/extras/timeline.jl index 8b7e0f699..c21e6bcfa 100644 --- a/src/extras/timeline.jl +++ b/src/extras/timeline.jl @@ -1,5 +1,8 @@ module Timeline +using Requires +export @range, mark + module NVTXT const LOG_FILE=Ref{IOStream}() const SHOULD_LOG=Ref{Bool}(false) @@ -91,7 +94,42 @@ module NVTXT end end # NVTXT -function range_push end -function mark end +_mark(msg) = NVTXT.mark(msg) +_push_range(msg) = NVTXT.push_range(msg) +_pop_range() = NVTXT.pop_range() +_start_range(msg) = NVTXT.start_range(msg) +_end_range(r) = NVTXT.end_range(r) + +@init @require CUDAnative="be33ccc6-a3ff-5ff2-a52e-74243cff1e17" begin + # replace implementations + import CUDAnative.NVTX + + _mark(msg) = NVTX.mark(msg) + _push_range(msg) = NVTX.push_range(msg) + _pop_range() = NVTX.pop_range() + _start_range(msg) = NVTX.start_range(msg) + _end_range(r) = NVTX.end_range(r) +end + +import Base: invokelatest +mark(msg) = invokelatest(_mark, msg) +push_range(msg) = invokelatest(_push_range, msg) +pop_range() = invokelatest(_pop_range) +start_range(msg) = invokelatest(_start_range, msg) +end_range(r) = invokelatest(_end_range, r) + +""" + @range "msg" ex +Create a new range and execute `ex`. The range is popped automatically afterwards. +See also: [`range`](@ref) +""" +macro range(msg, ex) + quote + local range = $start_range($(esc(msg))) + local ret = $(esc(ex)) + $end_range(range) + ret + end +end -end \ No newline at end of file +end